You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ni...@apache.org on 2013/02/10 03:13:21 UTC
[6/6] git commit: GIRAPH-470 (tavoaqp via nitay)
Updated Branches:
refs/heads/trunk a6cb05bcb -> ab64a4d0e
GIRAPH-470 (tavoaqp via nitay)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/ab64a4d0
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/ab64a4d0
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/ab64a4d0
Branch: refs/heads/trunk
Commit: ab64a4d0e4487ad7093c5be89088980c1f22019e
Parents: a6cb05b
Author: Nitay Joffe <ni...@gmail.com>
Authored: Sat Feb 9 21:12:02 2013 -0500
Committer: Nitay Joffe <ni...@gmail.com>
Committed: Sat Feb 9 21:12:25 2013 -0500
----------------------------------------------------------------------
CHANGELOG | 2 +
.../src/main/java/org/apache/giraph/Algorithm.java | 41 ++
.../main/java/org/apache/giraph/GiraphRunner.java | 1 -
.../giraph/examples/AggregatorsTestVertex.java | 134 ----
.../java/org/apache/giraph/examples/Algorithm.java | 41 --
.../giraph/examples/ConnectedComponentsVertex.java | 96 ---
.../giraph/examples/GeneratedVertexReader.java | 92 ---
.../org/apache/giraph/examples/IdentityVertex.java | 45 --
.../LongDoubleFloatDoubleTextInputFormat.java | 93 ---
...lizingLongDoubleFloatDoubleTextInputFormat.java | 126 ----
.../examples/PartitionContextTestVertex.java | 115 ----
.../apache/giraph/examples/RandomWalkVertex.java | 135 ----
.../examples/RandomWalkWithRestartVertex.java | 84 ---
.../giraph/examples/RandomWalkWorkerContext.java | 159 -----
.../giraph/examples/SimpleAggregatorWriter.java | 71 --
.../giraph/examples/SimpleCheckpointVertex.java | 287 --------
.../giraph/examples/SimpleCombinerVertex.java | 65 --
.../apache/giraph/examples/SimpleFailVertex.java | 69 --
.../giraph/examples/SimpleInDegreeCountVertex.java | 53 --
...SimpleLongDoubleDoubleDoubleIdentityVertex.java | 32 -
.../giraph/examples/SimpleMasterComputeVertex.java | 105 ---
.../apache/giraph/examples/SimpleMsgVertex.java | 62 --
.../giraph/examples/SimpleMutateGraphVertex.java | 198 ------
.../examples/SimpleOutDegreeCountVertex.java | 43 --
.../giraph/examples/SimplePageRankVertex.java | 253 -------
.../giraph/examples/SimpleShortestPathsVertex.java | 83 ---
.../giraph/examples/SimpleSuperstepVertex.java | 155 -----
.../examples/SimpleTextVertexOutputFormat.java | 55 --
.../examples/SimpleTriangleClosingVertex.java | 151 ----
.../examples/SimpleVertexWithWorkerContext.java | 197 ------
.../org/apache/giraph/examples/VerifyMessage.java | 222 ------
...exWithDoubleValueFloatEdgeTextOutputFormat.java | 57 --
.../org/apache/giraph/examples/package-info.java | 21 -
.../src/test/java/org/apache/giraph/BspCase.java | 6 +-
.../java/org/apache/giraph/TestAutoCheckpoint.java | 88 ---
.../test/java/org/apache/giraph/TestBspBasic.java | 527 ---------------
.../org/apache/giraph/TestGraphPartitioner.java | 162 -----
.../org/apache/giraph/TestManualCheckpoint.java | 115 ----
.../java/org/apache/giraph/TestMutateGraph.java | 64 --
.../org/apache/giraph/TestNotEnoughMapTasks.java | 75 --
.../org/apache/giraph/TestPartitionContext.java | 72 --
.../aggregators/TestAggregatorsHandling.java | 197 ------
.../examples/ConnectedComponentsVertexTest.java | 122 ----
.../giraph/examples/MinimumIntCombinerTest.java | 43 --
.../examples/RandomWalkWithRestartVertexTest.java | 131 ----
.../examples/SimpleShortestPathsVertexTest.java | 164 -----
.../examples/SimpleTriangleClosingVertexTest.java | 93 ---
.../org/apache/giraph/examples/TestPageRank.java | 104 ---
.../examples/TryMultiIpcBindingPortsTest.java | 126 ----
.../org/apache/giraph/vertex/TestVertexTypes.java | 209 ------
giraph-examples/pom.xml | 322 +++++++++
giraph-examples/src/main/assembly/compile.xml | 39 ++
.../giraph/examples/AggregatorsTestVertex.java | 134 ++++
.../java/org/apache/giraph/examples/Algorithm.java | 41 ++
.../giraph/examples/ConnectedComponentsVertex.java | 96 +++
.../giraph/examples/GeneratedVertexReader.java | 92 +++
.../org/apache/giraph/examples/IdentityVertex.java | 45 ++
.../LongDoubleFloatDoubleTextInputFormat.java | 93 +++
...lizingLongDoubleFloatDoubleTextInputFormat.java | 126 ++++
.../examples/PartitionContextTestVertex.java | 115 ++++
.../apache/giraph/examples/RandomWalkVertex.java | 135 ++++
.../examples/RandomWalkWithRestartVertex.java | 84 +++
.../giraph/examples/RandomWalkWorkerContext.java | 159 +++++
.../giraph/examples/SimpleAggregatorWriter.java | 71 ++
.../giraph/examples/SimpleCheckpointVertex.java | 287 ++++++++
.../giraph/examples/SimpleCombinerVertex.java | 65 ++
.../apache/giraph/examples/SimpleFailVertex.java | 69 ++
.../giraph/examples/SimpleInDegreeCountVertex.java | 53 ++
...SimpleLongDoubleDoubleDoubleIdentityVertex.java | 32 +
.../giraph/examples/SimpleMasterComputeVertex.java | 105 +++
.../apache/giraph/examples/SimpleMsgVertex.java | 62 ++
.../giraph/examples/SimpleMutateGraphVertex.java | 198 ++++++
.../examples/SimpleOutDegreeCountVertex.java | 43 ++
.../giraph/examples/SimplePageRankVertex.java | 253 +++++++
.../giraph/examples/SimpleShortestPathsVertex.java | 83 +++
.../giraph/examples/SimpleSuperstepVertex.java | 155 +++++
.../examples/SimpleTextVertexOutputFormat.java | 55 ++
.../examples/SimpleTriangleClosingVertex.java | 151 ++++
.../examples/SimpleVertexWithWorkerContext.java | 197 ++++++
.../org/apache/giraph/examples/VerifyMessage.java | 222 ++++++
...exWithDoubleValueFloatEdgeTextOutputFormat.java | 57 ++
.../org/apache/giraph/examples/package-info.java | 21 +
.../java/org/apache/giraph/TestAutoCheckpoint.java | 88 +++
.../test/java/org/apache/giraph/TestBspBasic.java | 527 +++++++++++++++
.../org/apache/giraph/TestGraphPartitioner.java | 162 +++++
.../org/apache/giraph/TestManualCheckpoint.java | 115 ++++
.../java/org/apache/giraph/TestMutateGraph.java | 64 ++
.../org/apache/giraph/TestNotEnoughMapTasks.java | 75 ++
.../org/apache/giraph/TestPartitionContext.java | 72 ++
.../aggregators/TestAggregatorsHandling.java | 197 ++++++
.../examples/ConnectedComponentsVertexTest.java | 122 ++++
.../giraph/examples/MinimumIntCombinerTest.java | 43 ++
.../examples/RandomWalkWithRestartVertexTest.java | 131 ++++
.../examples/SimpleShortestPathsVertexTest.java | 164 +++++
.../examples/SimpleTriangleClosingVertexTest.java | 93 +++
.../org/apache/giraph/examples/TestPageRank.java | 104 +++
.../examples/TryMultiIpcBindingPortsTest.java | 126 ++++
.../org/apache/giraph/vertex/TestVertexTypes.java | 209 ++++++
pom.xml | 1 +
99 files changed, 6000 insertions(+), 5594 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index c060209..d339945 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-470 (tavoaqp via nitay)
+
GIRAPH-504: Create PartitionContext (majakabiljo)
GIRAPH-499: Giraph should not reserve minimum reduce slot memory 1024 since we never use it (ereisman)
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/Algorithm.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/Algorithm.java b/giraph-core/src/main/java/org/apache/giraph/Algorithm.java
new file mode 100644
index 0000000..cdc9cba
--- /dev/null
+++ b/giraph-core/src/main/java/org/apache/giraph/Algorithm.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.giraph;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * This annotation should be used to annotate built-in algorithms.
+ */
+@Retention(value = RetentionPolicy.RUNTIME)
+@Target(value = ElementType.TYPE)
+public @interface Algorithm {
+ /**
+ * Name of the algorithm.
+ */
+ String name();
+
+ /**
+ * Short description of algorithm which is going to be presented to the user.
+ */
+ String description() default "";
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java b/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
index b6a6113..1516fff 100644
--- a/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
+++ b/giraph-core/src/main/java/org/apache/giraph/GiraphRunner.java
@@ -27,7 +27,6 @@ import org.apache.commons.cli.Options;
import org.apache.giraph.aggregators.AggregatorWriter;
import org.apache.giraph.combiner.Combiner;
import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.examples.Algorithm;
import org.apache.giraph.job.GiraphJob;
import org.apache.giraph.job.GiraphTypeValidator;
import org.apache.giraph.io.EdgeInputFormat;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/AggregatorsTestVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/AggregatorsTestVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/AggregatorsTestVertex.java
deleted file mode 100644
index c00fda0..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/AggregatorsTestVertex.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.aggregators.LongSumAggregator;
-import org.apache.giraph.master.DefaultMasterCompute;
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.io.IOException;
-
-/** Vertex which uses aggrergators. To be used for testing. */
-public class AggregatorsTestVertex extends
- EdgeListVertex<LongWritable, DoubleWritable, FloatWritable,
- DoubleWritable> {
-
- /** Name of regular aggregator */
- private static final String REGULAR_AGG = "regular";
- /** Name of persistent aggregator */
- private static final String PERSISTENT_AGG = "persistent";
- /** Name of master overwriting aggregator */
- private static final String MASTER_WRITE_AGG = "master";
- /** Value which master compute will use */
- private static final long MASTER_VALUE = 12345;
- /** Prefix for name of aggregators in array */
- private static final String ARRAY_PREFIX_AGG = "array";
- /** Number of aggregators to use in array */
- private static final int NUM_OF_AGGREGATORS_IN_ARRAY = 100;
-
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws IOException {
- long superstep = getSuperstep();
-
- LongWritable myValue = new LongWritable(1L << superstep);
- aggregate(REGULAR_AGG, myValue);
- aggregate(PERSISTENT_AGG, myValue);
-
- long nv = getTotalNumVertices();
- if (superstep > 0) {
- assertEquals(nv * (1L << (superstep - 1)),
- ((LongWritable) getAggregatedValue(REGULAR_AGG)).get());
- } else {
- assertEquals(0,
- ((LongWritable) getAggregatedValue(REGULAR_AGG)).get());
- }
- assertEquals(nv * ((1L << superstep) - 1),
- ((LongWritable) getAggregatedValue(PERSISTENT_AGG)).get());
- assertEquals(MASTER_VALUE * (1L << superstep),
- ((LongWritable) getAggregatedValue(MASTER_WRITE_AGG)).get());
-
- for (int i = 0; i < NUM_OF_AGGREGATORS_IN_ARRAY; i++) {
- aggregate(ARRAY_PREFIX_AGG + i, new LongWritable((superstep + 1) * i));
- assertEquals(superstep * getTotalNumVertices() * i,
- ((LongWritable) getAggregatedValue(ARRAY_PREFIX_AGG + i)).get());
- }
-
- if (getSuperstep() == 10) {
- voteToHalt();
- }
- }
-
- /** Master compute which uses aggregators. To be used for testing. */
- public static class AggregatorsTestMasterCompute extends
- DefaultMasterCompute {
- @Override
- public void compute() {
- long superstep = getSuperstep();
-
- LongWritable myValue =
- new LongWritable(MASTER_VALUE * (1L << superstep));
- setAggregatedValue(MASTER_WRITE_AGG, myValue);
-
- long nv = getTotalNumVertices();
- if (superstep > 0) {
- assertEquals(nv * (1L << (superstep - 1)),
- ((LongWritable) getAggregatedValue(REGULAR_AGG)).get());
- } else {
- assertEquals(0,
- ((LongWritable) getAggregatedValue(REGULAR_AGG)).get());
- }
- assertEquals(nv * ((1L << superstep) - 1),
- ((LongWritable) getAggregatedValue(PERSISTENT_AGG)).get());
-
- for (int i = 0; i < NUM_OF_AGGREGATORS_IN_ARRAY; i++) {
- assertEquals(superstep * getTotalNumVertices() * i,
- ((LongWritable) getAggregatedValue(ARRAY_PREFIX_AGG + i)).get());
- }
- }
-
- @Override
- public void initialize() throws InstantiationException,
- IllegalAccessException {
- registerAggregator(REGULAR_AGG, LongSumAggregator.class);
- registerPersistentAggregator(PERSISTENT_AGG,
- LongSumAggregator.class);
- registerAggregator(MASTER_WRITE_AGG, LongSumAggregator.class);
-
- for (int i = 0; i < NUM_OF_AGGREGATORS_IN_ARRAY; i++) {
- registerAggregator(ARRAY_PREFIX_AGG + i, LongSumAggregator.class);
- }
- }
- }
-
- /**
- * Throws exception if values are not equal.
- *
- * @param expected Expected value
- * @param actual Actual value
- */
- private static void assertEquals(long expected, long actual) {
- if (expected != actual) {
- throw new RuntimeException("expected: " + expected +
- ", actual: " + actual);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/Algorithm.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/Algorithm.java b/giraph-core/src/main/java/org/apache/giraph/examples/Algorithm.java
deleted file mode 100644
index 57d5011..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/Algorithm.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * This annotation should be used to annotate built-in algorithms.
- */
-@Retention(value = RetentionPolicy.RUNTIME)
-@Target(value = ElementType.TYPE)
-public @interface Algorithm {
- /**
- * Name of the algorithm.
- */
- String name();
-
- /**
- * Short description of algorithm which is going to be presented to the user.
- */
- String description() default "";
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/ConnectedComponentsVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/ConnectedComponentsVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/ConnectedComponentsVertex.java
deleted file mode 100644
index b5c90e5..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/ConnectedComponentsVertex.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.vertex.IntIntNullIntVertex;
-import org.apache.hadoop.io.IntWritable;
-
-import java.io.IOException;
-
-/**
- * Implementation of the HCC algorithm that identifies connected components and
- * assigns each vertex its "component identifier" (the smallest vertex id
- * in the component)
- *
- * The idea behind the algorithm is very simple: propagate the smallest
- * vertex id along the edges to all vertices of a connected component. The
- * number of supersteps necessary is equal to the length of the maximum
- * diameter of all components + 1
- *
- * The original Hadoop-based variant of this algorithm was proposed by Kang,
- * Charalampos, Tsourakakis and Faloutsos in
- * "PEGASUS: Mining Peta-Scale Graphs", 2010
- *
- * http://www.cs.cmu.edu/~ukang/papers/PegasusKAIS.pdf
- */
-@Algorithm(
- name = "Connected components",
- description = "Finds connected components of the graph"
-)
-public class ConnectedComponentsVertex extends IntIntNullIntVertex {
- /**
- * Propagates the smallest vertex id to all neighbors. Will always choose to
- * halt and only reactivate if a smaller id has been sent to it.
- *
- * @param messages Iterator of messages from the previous superstep.
- * @throws IOException
- */
- @Override
- public void compute(Iterable<IntWritable> messages) throws IOException {
- int currentComponent = getValue().get();
-
- // First superstep is special, because we can simply look at the neighbors
- if (getSuperstep() == 0) {
- for (IntWritable neighbor : getNeighbors()) {
- if (neighbor.get() < currentComponent) {
- currentComponent = neighbor.get();
- }
- }
- // Only need to send value if it is not the own id
- if (currentComponent != getValue().get()) {
- setValue(new IntWritable(currentComponent));
- for (IntWritable neighbor : getNeighbors()) {
- if (neighbor.get() > currentComponent) {
- sendMessage(new IntWritable(neighbor.get()), getValue());
- }
- }
- }
-
- voteToHalt();
- return;
- }
-
- boolean changed = false;
- // did we get a smaller id ?
- for (IntWritable message : messages) {
- int candidateComponent = message.get();
- if (candidateComponent < currentComponent) {
- currentComponent = candidateComponent;
- changed = true;
- }
- }
-
- // propagate new component id to the neighbors
- if (changed) {
- setValue(new IntWritable(currentComponent));
- sendMessageToAllEdges(getValue());
- }
- voteToHalt();
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/GeneratedVertexReader.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/GeneratedVertexReader.java b/giraph-core/src/main/java/org/apache/giraph/examples/GeneratedVertexReader.java
deleted file mode 100644
index 26f970d..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/GeneratedVertexReader.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.bsp.BspInputSplit;
-import org.apache.giraph.io.VertexReader;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-import java.io.IOException;
-
-/**
- * Used by GeneratedVertexInputFormat to read some generated data
- *
- * @param <I> Vertex index value
- * @param <V> Vertex value
- * @param <E> Edge value
- * @param <M> Message data
- */
-@SuppressWarnings("rawtypes")
-public abstract class GeneratedVertexReader<
- I extends WritableComparable, V extends Writable, E extends Writable,
- M extends Writable> implements VertexReader<I, V, E, M> {
- /** Vertices produced by this reader */
- public static final String READER_VERTICES =
- "GeneratedVertexReader.reader_vertices";
- /** Default vertices produced by this reader */
- public static final long DEFAULT_READER_VERTICES = 10;
- /** Reverse the order of the vertices? */
- public static final String REVERSE_ID_ORDER =
- "GeneratedVertexReader.reverseIdOrder";
- /** Default ordering is not reversed */
- public static final boolean DEAFULT_REVERSE_ID_ORDER = false;
- /** Records read so far */
- protected long recordsRead = 0;
- /** Total records to read (on this split alone) */
- protected long totalRecords = 0;
- /** The input split from initialize(). */
- protected BspInputSplit inputSplit = null;
- /** Reverse the id order? */
- protected boolean reverseIdOrder;
- /** Saved configuration */
- protected ImmutableClassesGiraphConfiguration configuration = null;
-
- /**
- * Default constructor for reflection.
- */
- public GeneratedVertexReader() {
- }
-
- @Override
- public final void initialize(InputSplit inputSplit,
- TaskAttemptContext context) throws IOException {
- configuration = new ImmutableClassesGiraphConfiguration(
- context.getConfiguration());
- totalRecords = configuration.getLong(
- GeneratedVertexReader.READER_VERTICES,
- GeneratedVertexReader.DEFAULT_READER_VERTICES);
- reverseIdOrder = configuration.getBoolean(
- GeneratedVertexReader.REVERSE_ID_ORDER,
- GeneratedVertexReader.DEAFULT_REVERSE_ID_ORDER);
- this.inputSplit = (BspInputSplit) inputSplit;
- }
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public final float getProgress() throws IOException {
- return recordsRead * 100.0f / totalRecords;
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/IdentityVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/IdentityVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/IdentityVertex.java
deleted file mode 100644
index 7000035..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/IdentityVertex.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-
-/**
- * User applications can subclass IdentityVertex, which
- * simply prints the results that have been read for testing IO related
- * jobs under any inputformat
- *
- * @param <I> Vertex index value
- * @param <V> Vertex value
- * @param <E> Edge value
- * @param <M> Message value
- */
-@SuppressWarnings("rawtypes")
-
-public abstract class IdentityVertex<I extends WritableComparable,
- V extends Writable, E extends Writable, M extends Writable>
- extends EdgeListVertex<I, V, E, M> {
-
- @Override
- public void compute(Iterable<M> messages) {
- voteToHalt();
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/LongDoubleFloatDoubleTextInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/LongDoubleFloatDoubleTextInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/examples/LongDoubleFloatDoubleTextInputFormat.java
deleted file mode 100644
index 100687b..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/LongDoubleFloatDoubleTextInputFormat.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.bsp.BspUtils;
-import org.apache.giraph.graph.DefaultEdge;
-import org.apache.giraph.graph.Edge;
-import org.apache.giraph.io.formats.TextVertexInputFormat;
-import org.apache.giraph.vertex.Vertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-import com.google.common.collect.Lists;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.regex.Pattern;
-
-/**
- * Simple text-based {@link org.apache.giraph.io.VertexInputFormat} for
- * unweighted graphs with long ids. Each line consists of: vertex neighbor1
- * neighbor2 ...
- */
-public class LongDoubleFloatDoubleTextInputFormat
- extends
- TextVertexInputFormat<LongWritable, DoubleWritable,
- FloatWritable, DoubleWritable> {
-
- @Override
- public TextVertexReader createVertexReader(InputSplit split,
- TaskAttemptContext context)
- throws IOException {
- return new LongDoubleFloatDoubleVertexReader();
- }
-
- /**
- * Vertex reader associated with {@link LongDoubleFloatDoubleTextInputFormat}.
- */
- public class LongDoubleFloatDoubleVertexReader extends
- TextVertexInputFormat.TextVertexReader {
- /** Separator of the vertex and neighbors */
- private final Pattern separator = Pattern.compile("[\t ]");
-
- @Override
- public Vertex<LongWritable, DoubleWritable, FloatWritable, DoubleWritable>
- getCurrentVertex() throws IOException, InterruptedException {
- Vertex<LongWritable, DoubleWritable, FloatWritable, DoubleWritable>
- vertex = BspUtils.<LongWritable, DoubleWritable,
- FloatWritable, DoubleWritable>createVertex(getContext()
- .getConfiguration());
-
- String[] tokens =
- separator.split(getRecordReader().getCurrentValue().toString());
- List<Edge<LongWritable, FloatWritable>> edges =
- Lists.newArrayListWithCapacity(tokens.length - 1);
- float weight = 1.0f / (tokens.length - 1);
- for (int n = 1; n < tokens.length; n++) {
- edges.add(new DefaultEdge<LongWritable, FloatWritable>(
- new LongWritable(Long.parseLong(tokens[n])),
- new FloatWritable(weight)));
- }
-
- LongWritable vertexId = new LongWritable(Long.parseLong(tokens[0]));
- vertex.initialize(vertexId, new DoubleWritable(), edges);
-
- return vertex;
- }
-
- @Override
- public boolean nextVertex() throws IOException, InterruptedException {
- return getRecordReader().nextKeyValue();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/NormalizingLongDoubleFloatDoubleTextInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/NormalizingLongDoubleFloatDoubleTextInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/examples/NormalizingLongDoubleFloatDoubleTextInputFormat.java
deleted file mode 100644
index 592d78a..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/NormalizingLongDoubleFloatDoubleTextInputFormat.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.bsp.BspUtils;
-import org.apache.giraph.graph.DefaultEdge;
-import org.apache.giraph.graph.Edge;
-import org.apache.giraph.io.formats.TextVertexInputFormat;
-import org.apache.giraph.vertex.Vertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-import com.google.common.collect.Lists;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.regex.Pattern;
-
-/**
- * Simple text-based {@link org.apache.giraph.io.VertexInputFormat} for
- * unweighted graphs with long ids. Each line consists of: vertex
- * neighbor1:weight1 neighbor2:weight2 ...
- */
-public class NormalizingLongDoubleFloatDoubleTextInputFormat
- extends
- TextVertexInputFormat<LongWritable, DoubleWritable,
- FloatWritable, DoubleWritable> {
-
- @Override
- public TextVertexReader createVertexReader(
- InputSplit split, TaskAttemptContext context) throws IOException {
- return new NormalizingLongDoubleFloatDoubleVertexReader();
- }
-
- /**
- * Vertex reader associated with {@link LongDoubleFloatDoubleTextInputFormat}.
- */
- public class NormalizingLongDoubleFloatDoubleVertexReader
- extends TextVertexInputFormat.TextVertexReader {
- /** Separator of the vertex and neighbors */
- private final Pattern edgeSeparator = Pattern.compile("\\s+");
- /** Separator of the edge id and edge weight */
- private final Pattern weightSeparator = Pattern.compile(":");
-
- @Override
- public Vertex<LongWritable, DoubleWritable,
- FloatWritable, DoubleWritable> getCurrentVertex()
- throws IOException, InterruptedException {
- Vertex<LongWritable, DoubleWritable,
- FloatWritable, DoubleWritable> vertex = BspUtils
- .<LongWritable, DoubleWritable,
- FloatWritable, DoubleWritable>createVertex(getContext()
- .getConfiguration());
-
- String[] tokens = edgeSeparator.split(getRecordReader()
- .getCurrentValue().toString());
- List<Edge<LongWritable, FloatWritable>> edges = Lists
- .newArrayListWithCapacity(tokens.length - 1);
- parse(tokens, edges);
- normalize(edges);
-
- LongWritable vertexId = new LongWritable(Long.parseLong(tokens[0]));
- vertex.initialize(vertexId, new DoubleWritable(), edges);
-
- return vertex;
- }
-
- /**
- * Parse a set of tokens into a map ID -> weight.
- * @param tokens The tokens to be parsed.
- * @param edges The map that will contain the result of the parsing.
- */
- void parse(String[] tokens,
- Collection<Edge<LongWritable, FloatWritable>> edges) {
- for (int n = 1; n < tokens.length; n++) {
- String[] parts = weightSeparator.split(tokens[n]);
- edges.add(new DefaultEdge<LongWritable, FloatWritable>(
- new LongWritable(Long.parseLong(parts[0])),
- new FloatWritable(Float.parseFloat(parts[1]))));
- }
- }
-
- /**
- * Normalize the edges with L1 normalization.
- * @param edges The edges to be normalized.
- */
- void normalize(Collection<Edge<LongWritable, FloatWritable>> edges) {
- if (edges == null || edges.size() == 0) {
- throw new IllegalArgumentException(
- "Cannot normalize an empy set of edges");
- }
- float normalizer = 0.0f;
- for (Edge<LongWritable, FloatWritable> edge : edges) {
- normalizer += edge.getValue().get();
- }
- for (Edge<LongWritable, FloatWritable> edge : edges) {
- edge.getValue().set(edge.getValue().get() / normalizer);
- }
- }
-
- @Override
- public boolean nextVertex() throws IOException, InterruptedException {
- return getRecordReader().nextKeyValue();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/PartitionContextTestVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/PartitionContextTestVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/PartitionContextTestVertex.java
deleted file mode 100644
index f86c323..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/PartitionContextTestVertex.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.partition.DefaultPartitionContext;
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.giraph.worker.DefaultWorkerContext;
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
-
-import java.io.IOException;
-
-/**
- * Vertex to test the functionality of PartitionContext
- */
-public class PartitionContextTestVertex extends
- EdgeListVertex<LongWritable, DoubleWritable, FloatWritable,
- DoubleWritable> {
- /** How many compute threads to use in the test */
- public static final int NUM_COMPUTE_THREADS = 10;
- /** How many vertices to create for the test */
- public static final int NUM_VERTICES = 100;
- /** How many partitions to have */
- public static final int NUM_PARTITIONS = 25;
-
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws IOException {
- TestPartitionContextPartitionContext partitionContext =
- (TestPartitionContextPartitionContext) getPartitionContext();
- partitionContext.counter++;
- if (getSuperstep() > 5) {
- voteToHalt();
- }
- }
-
- /**
- * PartitionContext for TestPartitionContext
- */
- public static class TestPartitionContextPartitionContext extends
- DefaultPartitionContext {
- /**
- * The counter should hold the number of vertices in this partition,
- * plus the current superstep
- */
- private long counter;
-
- @Override
- public void preSuperstep(WorkerContext workerContext) {
- counter =
- ((TestPartitionContextWorkerContext) workerContext).superstepCounter;
- }
-
- @Override
- public void postSuperstep(WorkerContext workerContext) {
- ((TestPartitionContextWorkerContext) workerContext).totalCounter +=
- counter;
- }
- }
-
- /**
- * WorkerContext for TestPartitionContext
- */
- public static class TestPartitionContextWorkerContext extends
- DefaultWorkerContext {
- /** Current superstep */
- private long superstepCounter;
- /**
- * This counter should hold the sum of PartitionContext's counters
- */
- private long totalCounter;
-
- @Override
- public void preSuperstep() {
- superstepCounter = getSuperstep();
- totalCounter = 0;
- }
-
- @Override
- public void postSuperstep() {
- assertEquals(totalCounter,
- NUM_PARTITIONS * superstepCounter + getTotalNumVertices());
- }
- }
-
- /**
- * Throws exception if values are not equal.
- *
- * @param expected Expected value
- * @param actual Actual value
- */
- private static void assertEquals(long expected, long actual) {
- if (expected != actual) {
- throw new RuntimeException("expected: " + expected +
- ", actual: " + actual);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkVertex.java
deleted file mode 100644
index 5669cf2..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkVertex.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.aggregators.DoubleSumAggregator;
-import org.apache.giraph.graph.Edge;
-import org.apache.giraph.master.DefaultMasterCompute;
-import org.apache.giraph.vertex.LongDoubleFloatDoubleEdgeListVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.log4j.Logger;
-
-import java.io.IOException;
-
-/**
- * Base class for executing a random walk on the graph
- */
-public abstract class RandomWalkVertex extends
- LongDoubleFloatDoubleEdgeListVertex {
- /** Configuration parameter for the number of supersteps to execute */
- static final String MAX_SUPERSTEPS = RandomWalkVertex.class.getName() +
- ".maxSupersteps";
- /** Configuration parameter for the teleportation probability */
- static final String TELEPORTATION_PROBABILITY = RandomWalkVertex.class
- .getName() + ".teleportationProbability";
- /** Name of aggregator for dangling nodes */
- static final String DANGLING = "dangling";
- /** Logger */
- private static final Logger LOG = Logger.getLogger(RandomWalkVertex.class);
- /** State probability of the vertex */
- protected final DoubleWritable d = new DoubleWritable();
-
- /**
- * Compute an initial probability distribution for the vertex.
- * @return The initial probability value.
- */
- protected abstract double initialProbability();
-
- /**
- * Perform a single step of a random walk computation.
- * @param messages Messages received in the previous step.
- * @param teleportationProbability Probability of teleporting to another
- * vertex.
- * @return The new probability distribution value.
- */
- protected abstract double recompute(Iterable<DoubleWritable> messages,
- double teleportationProbability);
-
- @Override
- public void compute(Iterable<DoubleWritable> messages) throws IOException {
- double stateProbability;
-
- if (getSuperstep() > 0) {
- stateProbability = recompute(messages, teleportationProbability());
- } else {
- stateProbability = initialProbability();
- }
- d.set(stateProbability);
- setValue(d);
-
- // Compute dangling node contribution for next superstep
- if (getNumEdges() == 0) {
- aggregate(DANGLING, d);
- }
-
- // Execute the algorithm as often as configured,
- // alternatively convergence could be checked via an Aggregator
- if (getSuperstep() < maxSupersteps()) {
- for (Edge<LongWritable, FloatWritable> edge : getEdges()) {
- double transitionProbability = stateProbability * edge.getValue().get();
- sendMessage(edge.getTargetVertexId(), new DoubleWritable(
- transitionProbability));
- }
- } else {
- voteToHalt();
- }
- }
-
- /**
- * Reads the number of supersteps to execute from the configuration
- * @return number of supersteps to execute
- */
- private int maxSupersteps() {
- return ((RandomWalkWorkerContext) getWorkerContext()).getMaxSupersteps();
- }
-
- /**
- * Reads the teleportation probability from the configuration
- * @return teleportation probability
- */
- protected double teleportationProbability() {
- return ((RandomWalkWorkerContext) getWorkerContext())
- .getTeleportationProbability();
- }
-
- /**
- * Master compute associated with {@link RandomWalkVertex}. It handles
- * dangling nodes.
- */
- public static class RandomWalkVertexMasterCompute extends
- DefaultMasterCompute {
- @Override
- public void compute() {
- // TODO This is a good place to implement halting by checking convergence.
- double danglingContribution =
- this.<DoubleWritable>getAggregatedValue(RandomWalkVertex.DANGLING)
- .get();
- LOG.info("[Superstep " + getSuperstep() + "] Dangling contribution = " +
- danglingContribution);
- }
-
- @Override
- public void initialize() throws InstantiationException,
- IllegalAccessException {
- registerAggregator(RandomWalkVertex.DANGLING, DoubleSumAggregator.class);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWithRestartVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWithRestartVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWithRestartVertex.java
deleted file mode 100644
index 8a689ed..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWithRestartVertex.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.utils.MathUtils;
-import org.apache.hadoop.io.DoubleWritable;
-
-/**
- * Executes "RandomWalkWithRestart", a random walk on the graph which is biased
- * towards a source vertex. The resulting probabilities of staying at a given
- * vertex can be interpreted as a measure of proximity to the source vertex.
- */
-public class RandomWalkWithRestartVertex extends RandomWalkVertex {
-
- /** Configuration parameter for the source vertex */
- static final String SOURCE_VERTEX = RandomWalkWithRestartVertex.class
- .getName() + ".sourceVertex";
-
- /**
- * Checks whether the currently executed vertex is the source vertex
- * @return is the currently executed vertex the source vertex?
- */
- private boolean isSourceVertex() {
- return ((RandomWalkWorkerContext) getWorkerContext()).isSource(getId()
- .get());
- }
-
- /**
- * Returns the number of source vertexes.
- * @return The number of source vertexes.
- */
- private int numSourceVertexes() {
- return ((RandomWalkWorkerContext) getWorkerContext()).numSources();
- }
-
- /**
- * Returns the cumulated probability from dangling nodes.
- * @return The cumulated probability from dangling nodes.
- */
- private double getDanglingProbability() {
- return this.<DoubleWritable>getAggregatedValue(RandomWalkVertex.DANGLING)
- .get();
- }
-
- /**
- * Start with a uniform distribution.
- * @return A uniform probability over all the vertexces.
- */
- @Override
- protected double initialProbability() {
- return 1.0 / getTotalNumVertices();
- }
-
- @Override
- protected double recompute(Iterable<DoubleWritable> transitionProbabilities,
- double teleportationProbability) {
- double stateProbability = MathUtils.sum(transitionProbabilities);
- // Add the contribution of dangling nodes (weakly preferential
- // implementation: dangling nodes redistribute uniformly)
- stateProbability += getDanglingProbability() / getTotalNumVertices();
- // The random walk might teleport back to one of the source vertexes
- stateProbability *= 1 - teleportationProbability;
- if (isSourceVertex()) {
- stateProbability += teleportationProbability / numSourceVertexes();
- }
- return stateProbability;
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWorkerContext.java b/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWorkerContext.java
deleted file mode 100644
index 5cff23f..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/RandomWalkWorkerContext.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.Set;
-
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
-
-import com.google.common.collect.ImmutableSet;
-
-/**
- * Worker context for random walks.
- */
-public class RandomWalkWorkerContext extends WorkerContext {
- /** Default maximum number of iterations */
- private static final int DEFAULT_MAX_SUPERSTEPS = 30;
- /** Default teleportation probability */
- private static final float DEFAULT_TELEPORTATION_PROBABILITY = 0.15f;
- /** Maximum number of iterations */
- private static int MAX_SUPERSTEPS;
- /** Teleportation probability */
- private static double TELEPORTATION_PROBABILITY;
- /** Preference vector */
- private static Set<Long> SOURCES;
-
- /** Configuration parameter for the source vertex */
- private static final String SOURCE_VERTEX = RandomWalkWithRestartVertex.class
- .getName() + ".sourceVertex";
-
- /** Logger */
- private static final Logger LOG = Logger
- .getLogger(RandomWalkWorkerContext.class);
-
- /**
- * @return The maximum number of iterations to perform.
- */
- public int getMaxSupersteps() {
- if (MAX_SUPERSTEPS == 0) {
- throw new IllegalStateException(
- RandomWalkWorkerContext.class.getSimpleName() +
- " was not initialized. Realunch your job " +
- "by setting the appropriate WorkerContext");
- }
- return MAX_SUPERSTEPS;
- }
-
- /**
- * @return The teleportation probability.
- */
- public double getTeleportationProbability() {
- if (TELEPORTATION_PROBABILITY == 0) {
- throw new IllegalStateException(
- RandomWalkWorkerContext.class.getSimpleName() +
- " was not initialized. Realunch your job " +
- "by setting the appropriate WorkerContext");
- }
- return TELEPORTATION_PROBABILITY;
- }
-
- /**
- * Checks if a vertex is a source.
- * @param id The vertex ID to check.
- * @return True if the vertex is a source in the preference vector.
- */
- public boolean isSource(long id) {
- return SOURCES.contains(id);
- }
-
- /**
- * @return The number of sources in the preference vector.
- */
- public int numSources() {
- return SOURCES.size();
- }
-
- /**
- * Initialize sources for Random Walk with Restart. First option
- * (preferential) is single source given from the command line as a parameter.
- * Second option is a file with a list of vertex IDs, one per line. In this
- * second case the preference vector is a uniform distribution over these
- * vertexes.
- * @param configuration
- * The configuration.
- */
- private void initializeSources(Configuration configuration) {
- ImmutableSet.Builder<Long> builder = ImmutableSet.builder();
- long sourceVertex = configuration.getLong(SOURCE_VERTEX, Long.MIN_VALUE);
- if (sourceVertex != Long.MIN_VALUE) {
- builder.add(sourceVertex);
- } else {
- Path sourceFile = null;
- try {
- sourceFile = DistributedCache.getLocalCacheFiles(configuration)[0];
- FileSystem fs = FileSystem.getLocal(configuration);
- BufferedReader in = new BufferedReader(new InputStreamReader(
- fs.open(sourceFile)));
- String line;
- while ((line = in.readLine()) != null) {
- builder.add(Long.parseLong(line));
- }
- in.close();
- } catch (IOException e) {
- e.printStackTrace();
- getContext().setStatus(
- "Could not load local cache files: " + sourceFile);
- LOG.error("Could not load local cache files: " + sourceFile);
- }
- }
- SOURCES = builder.build();
- }
-
- @Override
- public void preApplication() throws InstantiationException,
- IllegalAccessException {
- Configuration configuration = this.getContext().getConfiguration();
- MAX_SUPERSTEPS = configuration.getInt(RandomWalkVertex.MAX_SUPERSTEPS,
- DEFAULT_MAX_SUPERSTEPS);
- TELEPORTATION_PROBABILITY = configuration.getFloat(
- RandomWalkVertex.TELEPORTATION_PROBABILITY,
- DEFAULT_TELEPORTATION_PROBABILITY);
- initializeSources(configuration);
- }
-
- @Override
- public void preSuperstep() {
- }
-
- @Override
- public void postSuperstep() {
- }
-
- @Override
- public void postApplication() {
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleAggregatorWriter.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleAggregatorWriter.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleAggregatorWriter.java
deleted file mode 100644
index 1887621..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleAggregatorWriter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import java.io.IOException;
-import java.util.Map.Entry;
-
-import org.apache.giraph.aggregators.AggregatorWriter;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-
-/**
- * This is a simple example for an aggregator writer. After each superstep
- * the writer will persist the aggregator values to disk, by use of the
- * Writable interface. The file will be created on the current working
- * directory.
- */
-public class SimpleAggregatorWriter implements AggregatorWriter {
- /** Name of the file we wrote to */
- private static String FILENAME;
- /** Saved output stream to write to */
- private FSDataOutputStream output;
-
- public static String getFilename() {
- return FILENAME;
- }
-
- @SuppressWarnings("rawtypes")
- @Override
- public void initialize(Context context, long applicationAttempt)
- throws IOException {
- FILENAME = "aggregatedValues_" + applicationAttempt;
- Path p = new Path(FILENAME);
- FileSystem fs = FileSystem.get(context.getConfiguration());
- output = fs.create(p, true);
- }
-
- @Override
- public void writeAggregator(
- Iterable<Entry<String, Writable>> aggregatorMap,
- long superstep) throws IOException {
- for (Entry<String, Writable> entry : aggregatorMap) {
- entry.getValue().write(output);
- }
- output.flush();
- }
-
- @Override
- public void close() throws IOException {
- output.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java
deleted file mode 100644
index 337f30e..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCheckpointVertex.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.giraph.aggregators.LongSumAggregator;
-import org.apache.giraph.graph.DefaultEdge;
-import org.apache.giraph.graph.Edge;
-import org.apache.giraph.job.GiraphJob;
-import org.apache.giraph.io.formats.GeneratedVertexInputFormat;
-import org.apache.giraph.io.formats.IdWithValueTextOutputFormat;
-import org.apache.giraph.master.DefaultMasterCompute;
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.log4j.Logger;
-
-/**
- * An example that simply uses its id, value, and edges to compute new data
- * every iteration to verify that checkpoint restarting works. Fault injection
- * can also test automated checkpoint restarts.
- */
-public class SimpleCheckpointVertex implements Tool {
- /** Which superstep to cause the worker to fail */
- public static final int FAULTING_SUPERSTEP = 4;
- /** Vertex id to fault on */
- public static final long FAULTING_VERTEX_ID = 1;
- /** Dynamically set number of supersteps */
- public static final String SUPERSTEP_COUNT =
- "simpleCheckpointVertex.superstepCount";
- /** Should fault? */
- public static final String ENABLE_FAULT =
- "simpleCheckpointVertex.enableFault";
- /** Class logger */
- private static final Logger LOG =
- Logger.getLogger(SimpleCheckpointVertex.class);
- /** Configuration */
- private Configuration conf;
-
- /**
- * Actual computation.
- */
- public static class SimpleCheckpointComputation extends
- EdgeListVertex<LongWritable, IntWritable, FloatWritable, FloatWritable> {
- @Override
- public void compute(Iterable<FloatWritable> messages) {
- SimpleCheckpointVertexWorkerContext workerContext =
- (SimpleCheckpointVertexWorkerContext) getWorkerContext();
-
- boolean enableFault = workerContext.getEnableFault();
- int supersteps = workerContext.getSupersteps();
-
- if (enableFault && (getSuperstep() == FAULTING_SUPERSTEP) &&
- (getContext().getTaskAttemptID().getId() == 0) &&
- (getId().get() == FAULTING_VERTEX_ID)) {
- LOG.info("compute: Forced a fault on the first " +
- "attempt of superstep " +
- FAULTING_SUPERSTEP + " and vertex id " +
- FAULTING_VERTEX_ID);
- System.exit(-1);
- }
- if (getSuperstep() > supersteps) {
- voteToHalt();
- return;
- }
- long sumAgg = this.<LongWritable>getAggregatedValue(
- LongSumAggregator.class.getName()).get();
- LOG.info("compute: " + sumAgg);
- aggregate(LongSumAggregator.class.getName(),
- new LongWritable(getId().get()));
- LOG.info("compute: sum = " + sumAgg +
- " for vertex " + getId());
- float msgValue = 0.0f;
- for (FloatWritable message : messages) {
- float curMsgValue = message.get();
- msgValue += curMsgValue;
- LOG.info("compute: got msgValue = " + curMsgValue +
- " for vertex " + getId() +
- " on superstep " + getSuperstep());
- }
- int vertexValue = getValue().get();
- setValue(new IntWritable(vertexValue + (int) msgValue));
- LOG.info("compute: vertex " + getId() +
- " has value " + getValue() +
- " on superstep " + getSuperstep());
- for (Edge<LongWritable, FloatWritable> edge : getEdges()) {
- FloatWritable newEdgeValue = new FloatWritable(edge.getValue().get() +
- (float) vertexValue);
- Edge<LongWritable, FloatWritable> newEdge =
- new DefaultEdge<LongWritable, FloatWritable>(
- edge.getTargetVertexId(), newEdgeValue);
- LOG.info("compute: vertex " + getId() +
- " sending edgeValue " + edge.getValue() +
- " vertexValue " + vertexValue +
- " total " + newEdgeValue +
- " to vertex " + edge.getTargetVertexId() +
- " on superstep " + getSuperstep());
- addEdge(newEdge);
- sendMessage(edge.getTargetVertexId(), newEdgeValue);
- }
- }
- }
-
- /**
- * Worker context associated with {@link SimpleCheckpointVertex}.
- */
- public static class SimpleCheckpointVertexWorkerContext
- extends WorkerContext {
- /** Filename to indicate whether a fault was found */
- public static final String FAULT_FILE = "/tmp/faultFile";
- /** User can access this after the application finishes if local */
- private static long FINAL_SUM;
- /** Number of supersteps to run (6 by default) */
- private int supersteps = 6;
- /** Enable the fault at the particular vertex id and superstep? */
- private boolean enableFault = false;
-
- public static long getFinalSum() {
- return FINAL_SUM;
- }
-
- @Override
- public void preApplication()
- throws InstantiationException, IllegalAccessException {
- supersteps = getContext().getConfiguration()
- .getInt(SUPERSTEP_COUNT, supersteps);
- enableFault = getContext().getConfiguration()
- .getBoolean(ENABLE_FAULT, false);
- }
-
- @Override
- public void postApplication() {
- FINAL_SUM = this.<LongWritable>getAggregatedValue(
- LongSumAggregator.class.getName()).get();
- LOG.info("FINAL_SUM=" + FINAL_SUM);
- }
-
- @Override
- public void preSuperstep() {
- }
-
- @Override
- public void postSuperstep() { }
-
- public int getSupersteps() {
- return this.supersteps;
- }
-
- public boolean getEnableFault() {
- return this.enableFault;
- }
- }
-
- @Override
- public int run(String[] args) throws Exception {
- Options options = new Options();
- options.addOption("h", "help", false, "Help");
- options.addOption("v", "verbose", false, "Verbose");
- options.addOption("w",
- "workers",
- true,
- "Number of workers");
- options.addOption("s",
- "supersteps",
- true,
- "Supersteps to execute before finishing");
- options.addOption("w",
- "workers",
- true,
- "Minimum number of workers");
- options.addOption("o",
- "outputDirectory",
- true,
- "Output directory");
- HelpFormatter formatter = new HelpFormatter();
- if (args.length == 0) {
- formatter.printHelp(getClass().getName(), options, true);
- return 0;
- }
- CommandLineParser parser = new PosixParser();
- CommandLine cmd = parser.parse(options, args);
- if (cmd.hasOption('h')) {
- formatter.printHelp(getClass().getName(), options, true);
- return 0;
- }
- if (!cmd.hasOption('w')) {
- LOG.info("Need to choose the number of workers (-w)");
- return -1;
- }
- if (!cmd.hasOption('o')) {
- LOG.info("Need to set the output directory (-o)");
- return -1;
- }
-
- GiraphJob bspJob = new GiraphJob(getConf(), getClass().getName());
- bspJob.getConfiguration().setVertexClass(SimpleCheckpointComputation.class);
- bspJob.getConfiguration().setVertexInputFormatClass(
- GeneratedVertexInputFormat.class);
- bspJob.getConfiguration().setVertexOutputFormatClass(
- IdWithValueTextOutputFormat.class);
- bspJob.getConfiguration().setWorkerContextClass(
- SimpleCheckpointVertexWorkerContext.class);
- bspJob.getConfiguration().setMasterComputeClass(
- SimpleCheckpointVertexMasterCompute.class);
- int minWorkers = Integer.parseInt(cmd.getOptionValue('w'));
- int maxWorkers = Integer.parseInt(cmd.getOptionValue('w'));
- bspJob.getConfiguration().setWorkerConfiguration(
- minWorkers, maxWorkers, 100.0f);
-
- FileOutputFormat.setOutputPath(bspJob.getInternalJob(),
- new Path(cmd.getOptionValue('o')));
- boolean verbose = false;
- if (cmd.hasOption('v')) {
- verbose = true;
- }
- if (cmd.hasOption('s')) {
- getConf().setInt(SUPERSTEP_COUNT,
- Integer.parseInt(cmd.getOptionValue('s')));
- }
- if (bspJob.run(verbose)) {
- return 0;
- } else {
- return -1;
- }
- }
-
- /**
- * Master compute associated with {@link SimpleCheckpointVertex}.
- * It registers required aggregators.
- */
- public static class SimpleCheckpointVertexMasterCompute extends
- DefaultMasterCompute {
- @Override
- public void initialize() throws InstantiationException,
- IllegalAccessException {
- registerAggregator(LongSumAggregator.class.getName(),
- LongSumAggregator.class);
- }
- }
-
- /**
- * Executable from the command line.
- *
- * @param args Command line args.
- * @throws Exception
- */
- public static void main(String[] args) throws Exception {
- System.exit(ToolRunner.run(new SimpleCheckpointVertex(), args));
- }
-
- @Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCombinerVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCombinerVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCombinerVertex.java
deleted file mode 100644
index be0f16c..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleCombinerVertex.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.log4j.Logger;
-
-/**
- * Test whether messages can go through a combiner.
- */
-public class SimpleCombinerVertex extends
- EdgeListVertex<LongWritable, IntWritable, FloatWritable, IntWritable> {
- /** Class logger */
- private static Logger LOG = Logger.getLogger(SimpleCombinerVertex.class);
-
- @Override
- public void compute(Iterable<IntWritable> messages) {
- if (getId().equals(new LongWritable(2))) {
- sendMessage(new LongWritable(1), new IntWritable(101));
- sendMessage(new LongWritable(1), new IntWritable(102));
- sendMessage(new LongWritable(1), new IntWritable(103));
- }
- if (!getId().equals(new LongWritable(1))) {
- voteToHalt();
- } else {
- // Check the messages
- int sum = 0;
- int num = 0;
- for (IntWritable message : messages) {
- sum += message.get();
- num++;
- }
- LOG.info("TestCombinerVertex: Received a sum of " + sum +
- " (should have 306 with a single message value)");
-
- if (num == 1 && sum == 306) {
- voteToHalt();
- }
- }
- if (getSuperstep() > 3) {
- throw new IllegalStateException(
- "TestCombinerVertex: Vertex 1 failed to receive " +
- "messages in time");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleFailVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleFailVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleFailVertex.java
deleted file mode 100644
index 521ed97..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleFailVertex.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.log4j.Logger;
-
-/**
- * Vertex to allow unit testing of failure detection
- */
-public class SimpleFailVertex extends EdgeListVertex<
- LongWritable, DoubleWritable, FloatWritable, DoubleWritable> {
- /** Class logger */
- private static Logger LOG = Logger.getLogger(SimpleFailVertex.class);
- /** TODO: Change this behavior to WorkerContext */
- private static long SUPERSTEP = 0;
-
- @Override
- public void compute(Iterable<DoubleWritable> messages) {
- if (getSuperstep() >= 1) {
- double sum = 0;
- for (DoubleWritable message : messages) {
- sum += message.get();
- }
- DoubleWritable vertexValue =
- new DoubleWritable((0.15f / getTotalNumVertices()) + 0.85f * sum);
- setValue(vertexValue);
- if (getSuperstep() < 30) {
- if (getSuperstep() == 20) {
- if (getId().get() == 10L) {
- try {
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- LOG.info("Sleep interrupted ", e);
- }
- System.exit(1);
- } else if (getSuperstep() - SUPERSTEP > 10) {
- return;
- }
- }
- long edges = getNumEdges();
- sendMessageToAllEdges(
- new DoubleWritable(getValue().get() / edges));
- } else {
- voteToHalt();
- }
- SUPERSTEP = getSuperstep();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleInDegreeCountVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleInDegreeCountVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleInDegreeCountVertex.java
deleted file mode 100644
index e05da3e..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleInDegreeCountVertex.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.graph.Edge;
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * Simple function to return the out degree for each vertex.
- */
-@Algorithm(
- name = "Indegree Count"
-)
-public class SimpleInDegreeCountVertex extends EdgeListVertex<
- LongWritable, LongWritable, DoubleWritable, DoubleWritable> {
-
- @Override
- public void compute(Iterable<DoubleWritable> messages) {
- if (getSuperstep() == 0) {
- Iterable<Edge<LongWritable, DoubleWritable>> edges = getEdges();
- for (Edge<LongWritable, DoubleWritable> edge : edges) {
- sendMessage(edge.getTargetVertexId(), new DoubleWritable(1.0));
- }
- } else {
- long sum = 0;
- for (DoubleWritable message : messages) {
- sum++;
- }
- LongWritable vertexValue = getValue();
- vertexValue.set(sum);
- setValue(vertexValue);
- voteToHalt();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleLongDoubleDoubleDoubleIdentityVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleLongDoubleDoubleDoubleIdentityVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleLongDoubleDoubleDoubleIdentityVertex.java
deleted file mode 100644
index c7349d1..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleLongDoubleDoubleDoubleIdentityVertex.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.LongWritable;
-
-/**
- * A simple use of the Identity Vertex for taking care of Long, Double,
- * Double, Double type Inputformat Good for use with
- * io.LongDoubleDoubleAdjacencyListVertexInputFormat
- */
-
-public abstract class SimpleLongDoubleDoubleDoubleIdentityVertex extends
- IdentityVertex<LongWritable, DoubleWritable,
- DoubleWritable, DoubleWritable> { }
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMasterComputeVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMasterComputeVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMasterComputeVertex.java
deleted file mode 100644
index e481ff7..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMasterComputeVertex.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.aggregators.DoubleOverwriteAggregator;
-import org.apache.giraph.master.DefaultMasterCompute;
-import org.apache.giraph.vertex.LongDoubleFloatDoubleVertex;
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.log4j.Logger;
-
-/**
- * Demonstrates a computation with a centralized part implemented via a
- * MasterCompute.
- */
-public class SimpleMasterComputeVertex extends LongDoubleFloatDoubleVertex {
- /** Aggregator to get values from the master to the workers */
- public static final String SMC_AGG = "simplemastercompute.aggregator";
- /** Logger */
- private static final Logger LOG =
- Logger.getLogger(SimpleMasterComputeVertex.class);
-
- @Override
- public void compute(Iterable<DoubleWritable> messages) {
- double oldSum = getSuperstep() == 0 ? 0 : getValue().get();
- double newValue = this.<DoubleWritable>getAggregatedValue(SMC_AGG).get();
- double newSum = oldSum + newValue;
- setValue(new DoubleWritable(newSum));
- SimpleMasterComputeWorkerContext workerContext =
- (SimpleMasterComputeWorkerContext) getWorkerContext();
- workerContext.setFinalSum(newSum);
- LOG.info("Current sum: " + newSum);
- }
-
- /**
- * Worker context used with {@link SimpleMasterComputeVertex}.
- */
- public static class SimpleMasterComputeWorkerContext
- extends WorkerContext {
- /** Final sum value for verification for local jobs */
- private static double FINAL_SUM;
-
- @Override
- public void preApplication()
- throws InstantiationException, IllegalAccessException {
- }
-
- @Override
- public void preSuperstep() {
- }
-
- @Override
- public void postSuperstep() {
- }
-
- @Override
- public void postApplication() {
- }
-
- public void setFinalSum(double sum) {
- FINAL_SUM = sum;
- }
-
- public static double getFinalSum() {
- return FINAL_SUM;
- }
- }
-
- /**
- * MasterCompute used with {@link SimpleMasterComputeVertex}.
- */
- public static class SimpleMasterCompute
- extends DefaultMasterCompute {
- @Override
- public void compute() {
- setAggregatedValue(SMC_AGG,
- new DoubleWritable(((double) getSuperstep()) / 2 + 1));
- if (getSuperstep() == 10) {
- haltComputation();
- }
- }
-
- @Override
- public void initialize() throws InstantiationException,
- IllegalAccessException {
- registerAggregator(SMC_AGG, DoubleOverwriteAggregator.class);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ab64a4d0/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMsgVertex.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMsgVertex.java b/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMsgVertex.java
deleted file mode 100644
index 8016490..0000000
--- a/giraph-core/src/main/java/org/apache/giraph/examples/SimpleMsgVertex.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.giraph.examples;
-
-import org.apache.giraph.vertex.EdgeListVertex;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.log4j.Logger;
-
-/**
- * Test whether messages can be sent and received by vertices.
- */
-public class SimpleMsgVertex extends
- EdgeListVertex<LongWritable, IntWritable, FloatWritable, IntWritable> {
- /** Class logger */
- private static Logger LOG = Logger.getLogger(SimpleMsgVertex.class);
- @Override
- public void compute(Iterable<IntWritable> messages) {
- if (getId().equals(new LongWritable(2))) {
- sendMessage(new LongWritable(1), new IntWritable(101));
- sendMessage(new LongWritable(1), new IntWritable(102));
- sendMessage(new LongWritable(1), new IntWritable(103));
- }
- if (!getId().equals(new LongWritable(1))) {
- voteToHalt();
- } else {
- /* Check the messages */
- int sum = 0;
- for (IntWritable message : messages) {
- sum += message.get();
- }
- LOG.info("compute: Received a sum of " + sum +
- " (will stop on 306)");
-
- if (sum == 306) {
- voteToHalt();
- }
- }
- if (getSuperstep() > 3) {
- System.err.println("compute: Vertex 1 failed to receive " +
- "messages in time");
- voteToHalt();
- }
- }
-}