You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ac...@apache.org on 2013/04/05 23:39:45 UTC
[2/2] git commit: updated refs/heads/trunk to 84cdd9f
GIRAPH-564: Input/output formats and readers/writers should implement
ImmutableClassesGiraphConfigurable (aching)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/84cdd9f8
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/84cdd9f8
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/84cdd9f8
Branch: refs/heads/trunk
Commit: 84cdd9f839e1336f4ee92891a457b5d007920e4c
Parents: bf0d074
Author: Avery Ching <ac...@fb.com>
Authored: Thu Apr 4 13:39:52 2013 -0700
Committer: Avery Ching <ac...@fb.com>
Committed: Fri Apr 5 14:39:22 2013 -0700
----------------------------------------------------------------------
CHANGELOG | 3 +
.../io/accumulo/AccumuloVertexInputFormat.java | 7 +-
.../io/accumulo/AccumuloVertexOutputFormat.java | 11 +-
.../edgemarker/AccumuloEdgeInputFormat.java | 97 ++---
.../edgemarker/AccumuloEdgeOutputFormat.java | 79 ++--
.../giraph/aggregators/AggregatorWriter.java | 3 +-
.../giraph/aggregators/TextAggregatorWriter.java | 16 +-
.../apache/giraph/io/BasicVertexValueReader.java | 5 +-
.../java/org/apache/giraph/io/EdgeInputFormat.java | 9 +-
.../main/java/org/apache/giraph/io/EdgeReader.java | 25 +-
.../org/apache/giraph/io/GiraphInputFormat.java | 5 +-
.../apache/giraph/io/ReverseEdgeDuplicator.java | 5 +-
.../org/apache/giraph/io/VertexInputFormat.java | 14 +-
.../org/apache/giraph/io/VertexOutputFormat.java | 8 +-
.../java/org/apache/giraph/io/VertexReader.java | 25 +-
.../org/apache/giraph/io/VertexValueReader.java | 16 +-
.../java/org/apache/giraph/io/VertexWriter.java | 14 +-
.../AdjacencyListTextVertexInputFormat.java | 22 +-
.../AdjacencyListTextVertexOutputFormat.java | 5 +-
.../io/formats/IdWithValueTextOutputFormat.java | 13 +-
.../formats/IntNullReverseTextEdgeInputFormat.java | 1 +
.../io/formats/PseudoRandomEdgeInputFormat.java | 31 +-
.../io/formats/PseudoRandomVertexInputFormat.java | 42 +--
.../io/formats/SequenceFileVertexInputFormat.java | 7 +-
.../io/formats/SequenceFileVertexOutputFormat.java | 7 +-
.../giraph/io/formats/TextEdgeInputFormat.java | 30 +--
.../giraph/io/formats/TextVertexInputFormat.java | 26 +-
.../giraph/io/formats/TextVertexOutputFormat.java | 6 +-
.../giraph/io/iterables/EdgeReaderWrapper.java | 5 +-
.../giraph/io/iterables/VertexReaderWrapper.java | 5 +-
.../MultiThreadedSuperstepOutput.java | 6 +
.../SynchronizedSuperstepOutput.java | 3 +
.../giraph/utils/InMemoryVertexInputFormat.java | 66 ++--
.../org/apache/giraph/worker/BspServiceWorker.java | 3 +
.../giraph/worker/EdgeInputSplitsCallable.java | 3 +
.../giraph/worker/VertexInputSplitsCallable.java | 2 +
.../TestAdjacencyListTextVertexOutputFormat.java | 49 ++-
.../giraph/io/TestIdWithValueTextOutputFormat.java | 51 ++-
...DoubleDoubleAdjacencyListVertexInputFormat.java | 25 +-
...DoubleDoubleAdjacencyListVertexInputFormat.java | 13 +-
.../giraph/examples/GeneratedVertexReader.java | 15 +-
.../giraph/examples/SimpleAggregatorWriter.java | 6 +-
.../giraph/examples/SimplePageRankVertex.java | 19 +-
.../giraph/examples/SimpleSuperstepVertex.java | 18 +-
.../giraph/io/hbase/HBaseVertexInputFormat.java | 31 +-
.../giraph/io/hbase/HBaseVertexOutputFormat.java | 26 +-
.../io/hbase/TestHBaseRootMarkerVertextFormat.java | 333 ++++++++-------
.../io/hbase/edgemarker/TableEdgeInputFormat.java | 93 ++--
.../io/hcatalog/HCatalogEdgeInputFormat.java | 11 +-
.../io/hcatalog/HCatalogVertexInputFormat.java | 31 +-
.../io/hcatalog/HCatalogVertexOutputFormat.java | 2 +-
.../hive/input/edge/HiveEdgeInputFormat.java | 16 +-
.../giraph/hive/input/edge/HiveEdgeReader.java | 11 +-
.../hive/input/vertex/HiveVertexInputFormat.java | 15 +-
.../giraph/hive/input/vertex/HiveVertexReader.java | 11 +-
.../giraph/hive/output/HiveVertexOutputFormat.java | 13 +-
.../giraph/hive/output/HiveVertexWriter.java | 36 +-
57 files changed, 685 insertions(+), 765 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 0d5a671..cfb238e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-564: Input/output formats and readers/writers should implement
+ ImmutableClassesGiraphConfigurable (aching)
+
GIRAPH-604: Clean up benchmarks (majakabiljo)
GIRAPH-605: Worker crashes if its vertices have no edges when using edge input (majakabiljo)
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
index 28dc951..221fabd 100644
--- a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
+++ b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexInputFormat.java
@@ -17,6 +17,8 @@
*/
package org.apache.giraph.io.accumulo;
+import java.io.IOException;
+import java.util.List;
import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
@@ -30,9 +32,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-import java.util.List;
-
/**
* Class which wraps the AccumuloInputFormat. It's designed
* as an extension point to VertexInputFormat subclasses who wish
@@ -67,7 +66,7 @@ public abstract class AccumuloVertexInputFormat<
public abstract static class AccumuloVertexReader<
I extends WritableComparable,
V extends Writable, E extends Writable>
- implements VertexReader<I, V, E> {
+ extends VertexReader<I, V, E> {
/** Giraph configuration */
private ImmutableClassesGiraphConfiguration<I, V, E, Writable>
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
index 182afad..1927ed7 100644
--- a/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
+++ b/giraph-accumulo/src/main/java/org/apache/giraph/io/accumulo/AccumuloVertexOutputFormat.java
@@ -17,6 +17,7 @@
*/
package org.apache.giraph.io.accumulo;
+import java.io.IOException;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
import org.apache.accumulo.core.data.Mutation;
import org.apache.giraph.io.VertexOutputFormat;
@@ -28,8 +29,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-
-import java.io.IOException;
/**
*
* Class which wraps the AccumuloOutputFormat. It's designed
@@ -72,10 +71,10 @@ public abstract class AccumuloVertexOutputFormat<
* @param <E> edge type
*/
public abstract static class AccumuloVertexWriter<
- I extends WritableComparable,
- V extends Writable,
- E extends Writable>
- implements VertexWriter<I, V, E> {
+ I extends WritableComparable,
+ V extends Writable,
+ E extends Writable>
+ extends VertexWriter<I, V, E> {
/**
* task attempt context.
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
index 4e76b74..5855cfc 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeInputFormat.java
@@ -18,78 +18,75 @@
package org.apache.giraph.io.accumulo.edgemarker;
import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import java.util.regex.Pattern;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
-import org.apache.giraph.io.VertexReader;
-import org.apache.giraph.io.accumulo.AccumuloVertexInputFormat;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.EdgeFactory;
import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.giraph.io.VertexReader;
+import org.apache.giraph.io.accumulo.AccumuloVertexInputFormat;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-import java.util.List;
-import java.util.regex.Pattern;
-
/*
Example subclass which reads in Key/Value pairs to construct vertex objects.
*/
public class AccumuloEdgeInputFormat
- extends AccumuloVertexInputFormat<Text, Text, Text> {
-
- private static final Text uselessEdgeValue = new Text();
- private Configuration conf;
- public VertexReader<Text, Text, Text>
- createVertexReader(InputSplit split, TaskAttemptContext context)
- throws IOException {
- try {
+ extends AccumuloVertexInputFormat<Text, Text, Text> {
- return new AccumuloEdgeVertexReader(
- accumuloInputFormat.createRecordReader(split, context)) {
- };
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
+ private static final Text uselessEdgeValue = new Text();
+ public VertexReader<Text, Text, Text>
+ createVertexReader(InputSplit split, TaskAttemptContext context)
+ throws IOException {
+ try {
+ return new AccumuloEdgeVertexReader(
+ accumuloInputFormat.createRecordReader(split, context)) {
+ };
+ } catch (InterruptedException e) {
+ throw new IOException(e);
}
- /*
- Reader takes Key/Value pairs from the underlying input format.
- */
- public static class AccumuloEdgeVertexReader
- extends AccumuloVertexReader<Text, Text, Text> {
- public static final Pattern commaPattern = Pattern.compile("[,]");
+ }
+ /*
+ Reader takes Key/Value pairs from the underlying input format.
+ */
+ public static class AccumuloEdgeVertexReader
+ extends AccumuloVertexReader<Text, Text, Text> {
- public AccumuloEdgeVertexReader(RecordReader<Key, Value> recordReader) {
- super(recordReader);
- }
+ public static final Pattern commaPattern = Pattern.compile("[,]");
+ public AccumuloEdgeVertexReader(RecordReader<Key, Value> recordReader) {
+ super(recordReader);
+ }
- public boolean nextVertex() throws IOException, InterruptedException {
- return getRecordReader().nextKeyValue();
- }
- /*
- Each Key/Value contains the information needed to construct the vertices.
- */
- public Vertex<Text, Text, Text, ?> getCurrentVertex()
- throws IOException, InterruptedException {
- Key key = getRecordReader().getCurrentKey();
- Value value = getRecordReader().getCurrentValue();
- Vertex<Text, Text, Text, ?> vertex =
- getConfiguration().createVertex();
- Text vertexId = key.getRow();
- List<Edge<Text, Text>> edges = Lists.newLinkedList();
- String edge = new String(value.get());
- Text edgeId = new Text(edge);
- edges.add(EdgeFactory.create(edgeId, uselessEdgeValue));
- vertex.initialize(vertexId, new Text(), edges);
+ public boolean nextVertex() throws IOException, InterruptedException {
+ return getRecordReader().nextKeyValue();
+ }
+
+ /*
+ Each Key/Value contains the information needed to construct the vertices.
+ */
+ public Vertex<Text, Text, Text, ?> getCurrentVertex()
+ throws IOException, InterruptedException {
+ Key key = getRecordReader().getCurrentKey();
+ Value value = getRecordReader().getCurrentValue();
+ Vertex<Text, Text, Text, ?> vertex =
+ getConfiguration().createVertex();
+ Text vertexId = key.getRow();
+ List<Edge<Text, Text>> edges = Lists.newLinkedList();
+ String edge = new String(value.get());
+ Text edgeId = new Text(edge);
+ edges.add(EdgeFactory.create(edgeId, uselessEdgeValue));
+ vertex.initialize(vertexId, new Text(), edges);
- return vertex;
- }
+ return vertex;
}
+ }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
index 0937b84..f999497 100644
--- a/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
+++ b/giraph-accumulo/src/test/java/org/apache/giraph/io/accumulo/edgemarker/AccumuloEdgeOutputFormat.java
@@ -17,61 +17,60 @@
*/
package org.apache.giraph.io.accumulo.edgemarker;
+import java.io.IOException;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
-import org.apache.giraph.io.accumulo.AccumuloVertexOutputFormat;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexWriter;
+import org.apache.giraph.io.accumulo.AccumuloVertexOutputFormat;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/*
Example subclass for writing vertices back to Accumulo.
*/
public class AccumuloEdgeOutputFormat
- extends AccumuloVertexOutputFormat<Text, Text, Text> {
+ extends AccumuloVertexOutputFormat<Text, Text, Text> {
- public VertexWriter<Text, Text, Text>
- createVertexWriter(TaskAttemptContext context)
- throws IOException, InterruptedException {
- RecordWriter<Text, Mutation> writer =
- accumuloOutputFormat.getRecordWriter(context);
- String tableName = context.getConfiguration().get(OUTPUT_TABLE);
- if(tableName == null)
- throw new IOException("Forgot to set table name " +
- "using AccumuloVertexOutputFormat.OUTPUT_TABLE");
- return new AccumuloEdgeVertexWriter(writer, tableName);
- }
+ public VertexWriter<Text, Text, Text>
+ createVertexWriter(TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ RecordWriter<Text, Mutation> writer =
+ accumuloOutputFormat.getRecordWriter(context);
+ String tableName = getConf().get(OUTPUT_TABLE);
+ if(tableName == null)
+ throw new IOException("Forgot to set table name " +
+ "using AccumuloVertexOutputFormat.OUTPUT_TABLE");
+ return new AccumuloEdgeVertexWriter(writer, tableName);
+ }
- /*
- Wraps RecordWriter for writing Mutations back to the configured Accumulo Table.
- */
- public static class AccumuloEdgeVertexWriter
- extends AccumuloVertexWriter<Text, Text, Text> {
+ /*
+ Wraps RecordWriter for writing Mutations back to the configured Accumulo Table.
+ */
+ public static class AccumuloEdgeVertexWriter
+ extends AccumuloVertexWriter<Text, Text, Text> {
- private final Text CF = new Text("cf");
- private final Text PARENT = new Text("parent");
- private Text tableName;
+ private final Text CF = new Text("cf");
+ private final Text PARENT = new Text("parent");
+ private Text tableName;
- public AccumuloEdgeVertexWriter(
- RecordWriter<Text, Mutation> writer, String tableName) {
- super(writer);
- this.tableName = new Text(tableName);
- }
- /*
- Write back a mutation that adds a qualifier for 'parent' containing the vertex value
- as the cell value. Assume the vertex ID corresponds to a key.
- */
- public void writeVertex(Vertex<Text, Text, Text, ?> vertex)
- throws IOException, InterruptedException {
- RecordWriter<Text, Mutation> writer = getRecordWriter();
- Mutation mt = new Mutation(vertex.getId());
- mt.put(CF, PARENT, new Value(
- vertex.getValue().toString().getBytes()));
- writer.write(tableName, mt);
- }
+ public AccumuloEdgeVertexWriter(
+ RecordWriter<Text, Mutation> writer, String tableName) {
+ super(writer);
+ this.tableName = new Text(tableName);
+ }
+ /*
+ Write back a mutation that adds a qualifier for 'parent' containing the vertex value
+ as the cell value. Assume the vertex ID corresponds to a key.
+ */
+ public void writeVertex(Vertex<Text, Text, Text, ?> vertex)
+ throws IOException, InterruptedException {
+ RecordWriter<Text, Mutation> writer = getRecordWriter();
+ Mutation mt = new Mutation(vertex.getId());
+ mt.put(CF, PARENT, new Value(
+ vertex.getValue().toString().getBytes()));
+ writer.write(tableName, mt);
}
+ }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java
index c1c6678..7ffe75e 100644
--- a/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/AggregatorWriter.java
@@ -21,6 +21,7 @@ package org.apache.giraph.aggregators;
import java.io.IOException;
import java.util.Map.Entry;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Mapper.Context;
@@ -31,7 +32,7 @@ import org.apache.hadoop.mapreduce.Mapper.Context;
* passed to {@link AggregatorWriter#writeAggregator(Iterable, long)} as the
* superstep value to signal the end of computation.
*/
-public interface AggregatorWriter {
+public interface AggregatorWriter extends ImmutableClassesGiraphConfigurable {
/** Signal for last superstep */
int LAST_SUPERSTEP = -1;
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java
index ef43714..2c96496 100644
--- a/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/TextAggregatorWriter.java
@@ -18,11 +18,10 @@
package org.apache.giraph.aggregators;
+import com.google.common.base.Charsets;
import java.io.IOException;
import java.util.Map.Entry;
-
-import com.google.common.base.Charsets;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,7 +33,9 @@ import org.apache.hadoop.mapreduce.Mapper.Context;
* text and contains the aggregator name, the aggregator value and the
* aggregator class.
*/
-public class TextAggregatorWriter implements AggregatorWriter {
+public class TextAggregatorWriter
+ extends DefaultImmutableClassesGiraphConfigurable
+ implements AggregatorWriter {
/** The filename of the outputfile */
public static final String FILENAME =
"giraph.textAggregatorWriter.filename";
@@ -61,12 +62,11 @@ public class TextAggregatorWriter implements AggregatorWriter {
@Override
@SuppressWarnings("rawtypes")
public void initialize(Context context, long attempt) throws IOException {
- Configuration conf = context.getConfiguration();
- frequency = conf.getInt(FREQUENCY, NEVER);
- String filename = conf.get(FILENAME, DEFAULT_FILENAME);
+ frequency = getConf().getInt(FREQUENCY, NEVER);
+ String filename = getConf().get(FILENAME, DEFAULT_FILENAME);
if (frequency != NEVER) {
Path p = new Path(filename + "_" + attempt);
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = FileSystem.get(getConf());
if (fs.exists(p)) {
throw new RuntimeException("aggregatorWriter file already" +
" exists: " + p.getName());
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/BasicVertexValueReader.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/BasicVertexValueReader.java b/giraph-core/src/main/java/org/apache/giraph/io/BasicVertexValueReader.java
index 1aed963..310c39a 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/BasicVertexValueReader.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/BasicVertexValueReader.java
@@ -18,11 +18,10 @@
package org.apache.giraph.io;
+import java.io.IOException;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import java.io.IOException;
-
/**
* Abstract base class for VertexValueReader.
*
@@ -31,7 +30,7 @@ import java.io.IOException;
*/
public abstract class BasicVertexValueReader<I extends WritableComparable,
V extends Writable>
- implements VertexReader<I, V, Writable> {
+ extends VertexReader<I, V, Writable> {
/**
* User-defined method to extract the vertex id.
*
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/EdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/EdgeInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/EdgeInputFormat.java
index 87ea5a0..43cc7be 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/EdgeInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/EdgeInputFormat.java
@@ -18,6 +18,7 @@
package org.apache.giraph.io;
+import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -28,13 +29,17 @@ import java.io.IOException;
import java.util.List;
/**
- * Input format for reading single edges.
+ * Input format for reading single edges. Provides access to
+ * ImmutableClassesGiraphConfiguration.
*
* @param <I> Vertex id
* @param <E> Edge data
*/
public abstract class EdgeInputFormat<I extends WritableComparable,
- E extends Writable> implements GiraphInputFormat {
+ E extends Writable>
+ extends
+ DefaultImmutableClassesGiraphConfigurable<I, Writable, E, Writable>
+ implements GiraphInputFormat {
@Override
public abstract List<InputSplit> getSplits(
JobContext context, int minSplitCountHint) throws IOException,
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/EdgeReader.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/EdgeReader.java b/giraph-core/src/main/java/org/apache/giraph/io/EdgeReader.java
index f6dccdc..6d87bf2 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/EdgeReader.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/EdgeReader.java
@@ -18,14 +18,14 @@
package org.apache.giraph.io;
+import java.io.IOException;
+import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable;
import org.apache.giraph.edge.Edge;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/**
* Analogous to {@link RecordReader} for edges. Will read the edges
* from an input split.
@@ -34,7 +34,9 @@ import java.io.IOException;
* @param <E> Edge data
*/
@SuppressWarnings("rawtypes")
-public interface EdgeReader<I extends WritableComparable, E extends Writable> {
+public abstract class EdgeReader<I extends WritableComparable,
+ E extends Writable> extends DefaultImmutableClassesGiraphConfigurable<
+ I, Writable, E, Writable> {
/**
* Use the input split and context to setup reading the edges.
* Guaranteed to be called prior to any other function.
@@ -44,8 +46,9 @@ public interface EdgeReader<I extends WritableComparable, E extends Writable> {
* @throws java.io.IOException
* @throws InterruptedException
*/
- void initialize(InputSplit inputSplit, TaskAttemptContext context) throws
- IOException, InterruptedException;
+ public abstract void initialize(InputSplit inputSplit,
+ TaskAttemptContext context)
+ throws IOException, InterruptedException;
/**
* Read the next edge.
@@ -54,7 +57,7 @@ public interface EdgeReader<I extends WritableComparable, E extends Writable> {
* @throws IOException
* @throws InterruptedException
*/
- boolean nextEdge() throws IOException, InterruptedException;
+ public abstract boolean nextEdge() throws IOException, InterruptedException;
/**
* Get the current edge source id.
@@ -64,7 +67,8 @@ public interface EdgeReader<I extends WritableComparable, E extends Writable> {
* @throws IOException
* @throws InterruptedException
*/
- I getCurrentSourceId() throws IOException, InterruptedException;
+ public abstract I getCurrentSourceId()
+ throws IOException, InterruptedException;
/**
* Get the current edge.
@@ -74,14 +78,15 @@ public interface EdgeReader<I extends WritableComparable, E extends Writable> {
* @throws IOException
* @throws InterruptedException
*/
- Edge<I, E> getCurrentEdge() throws IOException, InterruptedException;
+ public abstract Edge<I, E> getCurrentEdge()
+ throws IOException, InterruptedException;
/**
* Close this {@link EdgeReader} to future operations.
*
* @throws IOException
*/
- void close() throws IOException;
+ public abstract void close() throws IOException;
/**
* How much of the input has the {@link EdgeReader} consumed i.e.
@@ -91,5 +96,5 @@ public interface EdgeReader<I extends WritableComparable, E extends Writable> {
* @throws IOException
* @throws InterruptedException
*/
- float getProgress() throws IOException, InterruptedException;
+ public abstract float getProgress() throws IOException, InterruptedException;
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/GiraphInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/GiraphInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/GiraphInputFormat.java
index 6b175a2..13efc93 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/GiraphInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/GiraphInputFormat.java
@@ -18,11 +18,10 @@
package org.apache.giraph.io;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
/**
* Common interface for {@link VertexInputFormat} and {@link EdgeInputFormat}.
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/ReverseEdgeDuplicator.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/ReverseEdgeDuplicator.java b/giraph-core/src/main/java/org/apache/giraph/io/ReverseEdgeDuplicator.java
index d6cb886..e9c113d 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/ReverseEdgeDuplicator.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/ReverseEdgeDuplicator.java
@@ -18,6 +18,7 @@
package org.apache.giraph.io;
+import java.io.IOException;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.EdgeFactory;
import org.apache.hadoop.io.Writable;
@@ -25,8 +26,6 @@ import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/**
* An EdgeReader that creates the opposite direction edge for each edge read.
* Used to create an undirected graph from a directed input.
@@ -36,7 +35,7 @@ import java.io.IOException;
* @param <E> Edge Value
*/
public class ReverseEdgeDuplicator<I extends WritableComparable,
- E extends Writable> implements EdgeReader<I, E> {
+ E extends Writable> extends EdgeReader<I, E> {
/** The underlying EdgeReader to wrap */
private final EdgeReader<I, E> baseReader;
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/VertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/VertexInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/VertexInputFormat.java
index 5aface5..b3f234f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/VertexInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/VertexInputFormat.java
@@ -18,20 +18,20 @@
package org.apache.giraph.io;
+import java.io.IOException;
+import java.util.List;
+import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-import java.util.List;
-
/**
* Use this to load data for a BSP application. Note that the InputSplit must
* also implement Writable. The InputSplits will determine the partitioning of
* vertices across the mappers, so keep that in consideration when implementing
- * getSplits().
+ * getSplits(). Provides access to ImmutableClassesGiraphConfiguration.
*
* @param <I> Vertex id
* @param <V> Vertex value
@@ -40,6 +40,7 @@ import java.util.List;
@SuppressWarnings("rawtypes")
public abstract class VertexInputFormat<I extends WritableComparable,
V extends Writable, E extends Writable>
+ extends DefaultImmutableClassesGiraphConfigurable<I, V, E, Writable>
implements GiraphInputFormat {
@Override
public abstract List<InputSplit> getSplits(
@@ -47,8 +48,9 @@ public abstract class VertexInputFormat<I extends WritableComparable,
throws IOException, InterruptedException;
/**
- * Create a vertex reader for a given split. The framework will call
- * {@link VertexReader#initialize(InputSplit, TaskAttemptContext)} before
+ * Create a vertex reader for a given split. Guaranteed to have been
+ * configured with setConf() prior to use. The framework will also call
+ * {@linkVertexReader#initialize(InputSplit, TaskAttemptContext)} before
* the split is used.
*
* @param split the split to be read
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/VertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/VertexOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/VertexOutputFormat.java
index 270d040..71eb665 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/VertexOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/VertexOutputFormat.java
@@ -20,6 +20,7 @@ package org.apache.giraph.io;
import java.io.IOException;
+import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -30,6 +31,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
/**
* Implement to output the graph after the computation. It is modeled
* directly after the Hadoop OutputFormat.
+ * ImmutableClassesGiraphConfiguration is available
*
* @param <I> Vertex index value
* @param <V> Vertex value
@@ -37,10 +39,12 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
*/
@SuppressWarnings("rawtypes")
public abstract class VertexOutputFormat<
- I extends WritableComparable, V extends Writable, E extends Writable> {
+ I extends WritableComparable, V extends Writable,
+ E extends Writable> extends
+ DefaultImmutableClassesGiraphConfigurable<I, V, E, Writable> {
/**
* Create a vertex writer for a given split. The framework will call
- * {@link VertexReader#initialize(InputSplit, TaskAttemptContext)} before
+ * {@link VertexWriter#initialize(TaskAttemptContext)} before
* the split is used.
*
* @param context the information about the task
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/VertexReader.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/VertexReader.java b/giraph-core/src/main/java/org/apache/giraph/io/VertexReader.java
index ef4ded6..3f6bb3f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/VertexReader.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/VertexReader.java
@@ -18,25 +18,26 @@
package org.apache.giraph.io;
+import java.io.IOException;
+import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable;
import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/**
- * Analogous to {@link RecordReader} for vertices. Will read the vertices
- * from an input split.
+ * Analogous to Hadoop's RecordReader for vertices. Will read the
+ * vertices from an input split.
*
* @param <I> Vertex id
* @param <V> Vertex data
* @param <E> Edge data
*/
@SuppressWarnings("rawtypes")
-public interface VertexReader<I extends WritableComparable,
- V extends Writable, E extends Writable> {
+public abstract class VertexReader<I extends WritableComparable,
+ V extends Writable, E extends Writable> extends
+ DefaultImmutableClassesGiraphConfigurable<I, V, E, Writable> {
/**
* Use the input split and context to setup reading the vertices.
* Guaranteed to be called prior to any other function.
@@ -46,7 +47,8 @@ public interface VertexReader<I extends WritableComparable,
* @throws IOException
* @throws InterruptedException
*/
- void initialize(InputSplit inputSplit, TaskAttemptContext context)
+ public abstract void initialize(InputSplit inputSplit,
+ TaskAttemptContext context)
throws IOException, InterruptedException;
/**
@@ -55,7 +57,8 @@ public interface VertexReader<I extends WritableComparable,
* @throws IOException
* @throws InterruptedException
*/
- boolean nextVertex() throws IOException, InterruptedException;
+ public abstract boolean nextVertex() throws IOException,
+ InterruptedException;
/**
* Get the current vertex.
@@ -65,7 +68,7 @@ public interface VertexReader<I extends WritableComparable,
* @throws IOException
* @throws InterruptedException
*/
- Vertex<I, V, E, ?> getCurrentVertex()
+ public abstract Vertex<I, V, E, ?> getCurrentVertex()
throws IOException, InterruptedException;
/**
@@ -73,7 +76,7 @@ public interface VertexReader<I extends WritableComparable,
*
* @throws IOException
*/
- void close() throws IOException;
+ public abstract void close() throws IOException;
/**
* How much of the input has the {@link VertexReader} consumed i.e.
@@ -83,5 +86,5 @@ public interface VertexReader<I extends WritableComparable,
* @throws IOException
* @throws InterruptedException
*/
- float getProgress() throws IOException, InterruptedException;
+ public abstract float getProgress() throws IOException, InterruptedException;
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/VertexValueReader.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/VertexValueReader.java b/giraph-core/src/main/java/org/apache/giraph/io/VertexValueReader.java
index 25e6093..0b06a4a 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/VertexValueReader.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/VertexValueReader.java
@@ -18,15 +18,13 @@
package org.apache.giraph.io;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import java.io.IOException;
import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/**
* Vertex reader for {@link org.apache.giraph.io.VertexValueInputFormat}.
*
@@ -35,16 +33,9 @@ import java.io.IOException;
*/
public abstract class VertexValueReader<I extends WritableComparable,
V extends Writable> extends BasicVertexValueReader<I, V> {
- /** Configuration. */
- private ImmutableClassesGiraphConfiguration<I, V, Writable, Writable>
- configuration;
-
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
throws IOException, InterruptedException {
- configuration =
- new ImmutableClassesGiraphConfiguration<I, V, Writable, Writable>(
- context.getConfiguration());
}
@Override
@@ -54,9 +45,4 @@ public abstract class VertexValueReader<I extends WritableComparable,
vertex.initialize(getCurrentVertexId(), getCurrentVertexValue());
return vertex;
}
-
- public ImmutableClassesGiraphConfiguration<I, V, Writable, Writable>
- getConf() {
- return configuration;
- }
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/VertexWriter.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/VertexWriter.java b/giraph-core/src/main/java/org/apache/giraph/io/VertexWriter.java
index 2aa3a71..a4285c1 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/VertexWriter.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/VertexWriter.java
@@ -19,7 +19,7 @@
package org.apache.giraph.io;
import java.io.IOException;
-
+import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -32,8 +32,10 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
* @param <E> Edge value
*/
@SuppressWarnings("rawtypes")
-public interface VertexWriter<I extends WritableComparable, V extends Writable,
- E extends Writable> extends SimpleVertexWriter<I, V, E> {
+public abstract class VertexWriter<I extends WritableComparable,
+ V extends Writable, E extends Writable>
+ extends DefaultImmutableClassesGiraphConfigurable<I, V, E, Writable>
+ implements SimpleVertexWriter<I, V, E> {
/**
* Use the context to setup writing the vertices.
* Guaranteed to be called prior to any other function.
@@ -42,8 +44,8 @@ public interface VertexWriter<I extends WritableComparable, V extends Writable,
* @throws IOException
* @throws InterruptedException
*/
- void initialize(TaskAttemptContext context) throws IOException,
- InterruptedException;
+ public abstract void initialize(TaskAttemptContext context)
+ throws IOException, InterruptedException;
/**
* Close this {@link VertexWriter} to future operations.
@@ -52,6 +54,6 @@ public interface VertexWriter<I extends WritableComparable, V extends Writable,
* @throws IOException
* @throws InterruptedException
*/
- void close(TaskAttemptContext context)
+ public abstract void close(TaskAttemptContext context)
throws IOException, InterruptedException;
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexInputFormat.java
index 7af4a71..faddc24 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexInputFormat.java
@@ -18,17 +18,15 @@
package org.apache.giraph.io.formats;
import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
import org.apache.giraph.edge.Edge;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-import java.util.List;
-
/**
* VertexReader that readers lines of text with vertices encoded as adjacency
* lists and converts each token to the correct type. For example, a graph
@@ -72,23 +70,13 @@ public abstract class AdjacencyListTextVertexInputFormat<I extends
*/
protected abstract class AdjacencyListTextVertexReader extends
TextVertexReaderFromEachLineProcessed<String[]> {
- /**
- * Cached configuration.
- */
- private Configuration conf;
-
/** Cached delimiter used for split */
private String splitValue = null;
-
- /**
- * Sanitizer from constructor.
- */
+ /** Sanitizer from constructor. */
private final LineSanitizer sanitizer;
-
/**
* Constructor without line sanitizer.
- *
*/
public AdjacencyListTextVertexReader() {
this(null);
@@ -107,8 +95,8 @@ public abstract class AdjacencyListTextVertexInputFormat<I extends
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
throws IOException, InterruptedException {
super.initialize(inputSplit, context);
- conf = context.getConfiguration();
- splitValue = conf.get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
+ splitValue =
+ getConf().get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexOutputFormat.java
index 5815403..f71ef25 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/AdjacencyListTextVertexOutputFormat.java
@@ -40,7 +40,6 @@ import java.io.IOException;
public class AdjacencyListTextVertexOutputFormat<I extends WritableComparable,
V extends Writable, E extends Writable>
extends TextVertexOutputFormat<I, V, E> {
-
/** Split delimiter */
public static final String LINE_TOKENIZE_VALUE = "output.delimiter";
/** Default split delimiter */
@@ -65,8 +64,8 @@ public class AdjacencyListTextVertexOutputFormat<I extends WritableComparable,
public void initialize(TaskAttemptContext context) throws IOException,
InterruptedException {
super.initialize(context);
- delimiter = context.getConfiguration()
- .get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
+ delimiter =
+ getConf().get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/IdWithValueTextOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/IdWithValueTextOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/IdWithValueTextOutputFormat.java
index fe4a1d5..6dd7468 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/IdWithValueTextOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/IdWithValueTextOutputFormat.java
@@ -19,15 +19,13 @@
package org.apache.giraph.io.formats;
+import java.io.IOException;
import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/**
* Write out Vertices' IDs and values, but not their edges nor edges' values.
* This is a useful output format when the final value of the vertex is
@@ -70,11 +68,10 @@ public class IdWithValueTextOutputFormat<I extends WritableComparable,
public void initialize(TaskAttemptContext context) throws IOException,
InterruptedException {
super.initialize(context);
- Configuration conf = context.getConfiguration();
- delimiter = conf
- .get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
- reverseOutput = conf
- .getBoolean(REVERSE_ID_AND_VALUE, REVERSE_ID_AND_VALUE_DEFAULT);
+ delimiter = getConf().get(
+ LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
+ reverseOutput = getConf().getBoolean(
+ REVERSE_ID_AND_VALUE, REVERSE_ID_AND_VALUE_DEFAULT);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/IntNullReverseTextEdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/IntNullReverseTextEdgeInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/IntNullReverseTextEdgeInputFormat.java
index 0270348..3aeb753 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/IntNullReverseTextEdgeInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/IntNullReverseTextEdgeInputFormat.java
@@ -42,6 +42,7 @@ public class IntNullReverseTextEdgeInputFormat
InputSplit split, TaskAttemptContext context) throws IOException {
EdgeReader<IntWritable, NullWritable> edgeReader =
super.createEdgeReader(split, context);
+ edgeReader.setConf(getConf());
return new ReverseEdgeDuplicator<IntWritable, NullWritable>(edgeReader);
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomEdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomEdgeInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomEdgeInputFormat.java
index cd454e3..87cadb4 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomEdgeInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomEdgeInputFormat.java
@@ -19,12 +19,16 @@
package org.apache.giraph.io.formats;
import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
import org.apache.giraph.bsp.BspInputSplit;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.io.EdgeInputFormat;
-import org.apache.giraph.io.EdgeReader;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.EdgeFactory;
+import org.apache.giraph.io.EdgeInputFormat;
+import org.apache.giraph.io.EdgeReader;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -32,12 +36,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.log4j.Logger;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
/**
* This {@link EdgeInputFormat} generates pseudo-random edges on the fly.
* As with {@link PseudoRandomVertexInputFormat}, the user specifies the
@@ -68,7 +66,7 @@ public class PseudoRandomEdgeInputFormat
* {@link EdgeReader} that generates pseudo-random edges.
*/
private static class PseudoRandomEdgeReader
- implements EdgeReader<LongWritable, DoubleWritable> {
+ extends EdgeReader<LongWritable, DoubleWritable> {
/** Logger. */
private static final Logger LOG =
Logger.getLogger(PseudoRandomEdgeReader.class);
@@ -93,18 +91,13 @@ public class PseudoRandomEdgeInputFormat
private int edgesPerVertex = -1;
/** BspInputSplit (used only for index). */
private BspInputSplit bspInputSplit;
- /** Saved configuration */
- private ImmutableClassesGiraphConfiguration configuration;
/** Helper for generating pseudo-random local edges. */
private PseudoRandomLocalEdgesHelper localEdgesHelper;
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
throws IOException, InterruptedException {
- configuration = new ImmutableClassesGiraphConfiguration(
- context.getConfiguration());
- aggregateVertices =
- configuration.getLong(
+ aggregateVertices = getConf().getLong(
PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, 0);
if (aggregateVertices <= 0) {
throw new IllegalArgumentException(
@@ -128,17 +121,17 @@ public class PseudoRandomEdgeInputFormat
"initialize: Got " + inputSplit.getClass() +
" instead of " + BspInputSplit.class);
}
- edgesPerVertex = configuration.getInt(
+ edgesPerVertex = getConf().getInt(
PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, 0);
if (edgesPerVertex <= 0) {
throw new IllegalArgumentException(
PseudoRandomInputFormatConstants.EDGES_PER_VERTEX + " <= 0");
}
- float minLocalEdgesRatio = configuration.getFloat(
+ float minLocalEdgesRatio = getConf().getFloat(
PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO,
PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT);
localEdgesHelper = new PseudoRandomLocalEdgesHelper(aggregateVertices,
- minLocalEdgesRatio, configuration);
+ minLocalEdgesRatio, getConf());
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomVertexInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomVertexInputFormat.java
index c261f72..dca0271 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomVertexInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/PseudoRandomVertexInputFormat.java
@@ -19,13 +19,17 @@
package org.apache.giraph.io.formats;
import com.google.common.collect.Sets;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
import org.apache.giraph.bsp.BspInputSplit;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.io.VertexInputFormat;
-import org.apache.giraph.io.VertexReader;
import org.apache.giraph.edge.EdgeFactory;
-import org.apache.giraph.graph.Vertex;
import org.apache.giraph.edge.VertexEdges;
+import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.io.VertexInputFormat;
+import org.apache.giraph.io.VertexReader;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -33,12 +37,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.log4j.Logger;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-
/**
* This VertexInputFormat is meant for large scale testing. It allows the user
* to create an input data source that a variable number of aggregate vertices
@@ -70,7 +68,7 @@ public class PseudoRandomVertexInputFormat extends
* Used by {@link PseudoRandomVertexInputFormat} to read
* pseudo-randomly generated data.
*/
- private static class PseudoRandomVertexReader implements
+ private static class PseudoRandomVertexReader extends
VertexReader<LongWritable, DoubleWritable, DoubleWritable> {
/** Logger. */
private static final Logger LOG =
@@ -87,8 +85,6 @@ public class PseudoRandomVertexInputFormat extends
private int edgesPerVertex = -1;
/** BspInputSplit (used only for index). */
private BspInputSplit bspInputSplit;
- /** Saved configuration */
- private ImmutableClassesGiraphConfiguration configuration;
/** Helper for generating pseudo-random local edges. */
private PseudoRandomLocalEdgesHelper localEdgesHelper;
@@ -101,11 +97,8 @@ public class PseudoRandomVertexInputFormat extends
@Override
public void initialize(InputSplit inputSplit,
TaskAttemptContext context) throws IOException {
- configuration = new ImmutableClassesGiraphConfiguration(
- context.getConfiguration());
- aggregateVertices =
- configuration.getLong(
- PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, 0);
+ aggregateVertices = getConf().getLong(
+ PseudoRandomInputFormatConstants.AGGREGATE_VERTICES, 0);
if (aggregateVertices <= 0) {
throw new IllegalArgumentException(
PseudoRandomInputFormatConstants.AGGREGATE_VERTICES + " <= 0");
@@ -128,17 +121,17 @@ public class PseudoRandomVertexInputFormat extends
"initialize: Got " + inputSplit.getClass() +
" instead of " + BspInputSplit.class);
}
- edgesPerVertex = configuration.getInt(
+ edgesPerVertex = getConf().getInt(
PseudoRandomInputFormatConstants.EDGES_PER_VERTEX, 0);
if (edgesPerVertex <= 0) {
throw new IllegalArgumentException(
PseudoRandomInputFormatConstants.EDGES_PER_VERTEX + " <= 0");
}
- float minLocalEdgesRatio = configuration.getFloat(
+ float minLocalEdgesRatio = getConf().getFloat(
PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO,
PseudoRandomInputFormatConstants.LOCAL_EDGES_MIN_RATIO_DEFAULT);
localEdgesHelper = new PseudoRandomLocalEdgesHelper(aggregateVertices,
- minLocalEdgesRatio, configuration);
+ minLocalEdgesRatio, getConf());
}
@Override
@@ -150,7 +143,7 @@ public class PseudoRandomVertexInputFormat extends
public Vertex<LongWritable, DoubleWritable, DoubleWritable, ?>
getCurrentVertex() throws IOException, InterruptedException {
Vertex<LongWritable, DoubleWritable, DoubleWritable, ?>
- vertex = configuration.createVertex();
+ vertex = getConf().createVertex();
long vertexId = startingVertexId + verticesRead;
// Seed on the vertex id to keep the vertex data the same when
// on different number of workers, but other parameters are the
@@ -160,7 +153,7 @@ public class PseudoRandomVertexInputFormat extends
// In order to save memory and avoid copying, we add directly to a
// VertexEdges instance.
VertexEdges<LongWritable, DoubleWritable> edges =
- configuration.createAndInitializeVertexEdges(edgesPerVertex);
+ getConf().createAndInitializeVertexEdges(edgesPerVertex);
Set<LongWritable> destVertices = Sets.newHashSet();
for (long i = 0; i < edgesPerVertex; ++i) {
LongWritable destVertexId = new LongWritable();
@@ -184,8 +177,7 @@ public class PseudoRandomVertexInputFormat extends
}
@Override
- public void close() throws IOException {
- }
+ public void close() throws IOException { }
@Override
public float getProgress() throws IOException {
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexInputFormat.java
index 3ffda1f..0f2d929 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexInputFormat.java
@@ -17,6 +17,8 @@
*/
package org.apache.giraph.io.formats;
+import java.io.IOException;
+import java.util.List;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexInputFormat;
import org.apache.giraph.io.VertexReader;
@@ -28,9 +30,6 @@ import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import java.io.IOException;
-import java.util.List;
-
/**
* Sequence file vertex input format based on {@link SequenceFileInputFormat}.
*
@@ -70,7 +69,7 @@ public class SequenceFileVertexInputFormat<I extends WritableComparable,
*/
public static class SequenceFileVertexReader<I extends WritableComparable,
V extends Writable, E extends Writable, X extends Vertex<I, V, E, ?>>
- implements VertexReader<I, V, E> {
+ extends VertexReader<I, V, E> {
/** Internal record reader from {@link SequenceFileInputFormat} */
private final RecordReader<I, X> recordReader;
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexOutputFormat.java
index 468e6bd..c4ed65c 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/SequenceFileVertexOutputFormat.java
@@ -17,9 +17,10 @@
*/
package org.apache.giraph.io.formats;
+import java.io.IOException;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexOutputFormat;
import org.apache.giraph.io.VertexWriter;
-import org.apache.giraph.graph.Vertex;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.JobContext;
@@ -28,8 +29,6 @@ import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import java.io.IOException;
-
/**
* Sequence file vertex output format. It allows to convert a vertex into a key
* and value pair of desired types, and output the pair into a sequence file.
@@ -92,7 +91,7 @@ public abstract class SequenceFileVertexOutputFormat<
* Vertex writer that converts a vertex into a key-value pair and writes
* the result into a sequence file for a context.
*/
- private class SequenceFileVertexWriter implements VertexWriter<I, V, E> {
+ private class SequenceFileVertexWriter extends VertexWriter<I, V, E> {
/**
* A record writer that will write into a sequence file initialized for
* a context.
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/TextEdgeInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/TextEdgeInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/TextEdgeInputFormat.java
index 0aae894..db3d5e0 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/TextEdgeInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/TextEdgeInputFormat.java
@@ -18,13 +18,13 @@
package org.apache.giraph.io.formats;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.io.EdgeInputFormat;
-import org.apache.giraph.io.EdgeReader;
+import java.io.IOException;
+import java.util.List;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.EdgeFactory;
+import org.apache.giraph.io.EdgeInputFormat;
+import org.apache.giraph.io.EdgeReader;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@@ -33,9 +33,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-import java.util.List;
-
/**
* Abstract class that users should subclass to use their own text based
* edge output format.
@@ -61,23 +58,16 @@ public abstract class TextEdgeInputFormat<I extends WritableComparable,
/**
* {@link EdgeReader} for {@link TextEdgeInputFormat}.
*/
- protected abstract class TextEdgeReader implements EdgeReader<I, E> {
+ protected abstract class TextEdgeReader extends EdgeReader<I, E> {
/** Internal line record reader */
private RecordReader<LongWritable, Text> lineRecordReader;
/** Context passed to initialize */
private TaskAttemptContext context;
- /**
- * Cached configuration. We don't care about vertex value and message type.
- */
- private ImmutableClassesGiraphConfiguration<I, NullWritable, E,
- NullWritable> conf;
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
throws IOException, InterruptedException {
this.context = context;
- conf = new ImmutableClassesGiraphConfiguration<I, NullWritable, E,
- NullWritable>(context.getConfiguration());
lineRecordReader = createLineRecordReader(inputSplit, context);
lineRecordReader.initialize(inputSplit, context);
}
@@ -130,16 +120,6 @@ public abstract class TextEdgeInputFormat<I extends WritableComparable,
protected TaskAttemptContext getContext() {
return context;
}
-
- /**
- * Get the configuration.
- *
- * @return Configuration for this reader
- */
- protected ImmutableClassesGiraphConfiguration<I, NullWritable, E,
- NullWritable> getConf() {
- return conf;
- }
}
/**
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexInputFormat.java
index e47bda3..b08e6f7 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexInputFormat.java
@@ -18,11 +18,12 @@
package org.apache.giraph.io.formats;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.giraph.io.VertexInputFormat;
-import org.apache.giraph.io.VertexReader;
+import java.io.IOException;
+import java.util.List;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.io.VertexInputFormat;
+import org.apache.giraph.io.VertexReader;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@@ -32,9 +33,6 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-import java.util.List;
-
/**
* Abstract class that users should subclass to use their own text based
* vertex input format.
@@ -84,21 +82,16 @@ public abstract class TextVertexInputFormat<I extends WritableComparable,
* need common exception handling while preprocessing, then extend
* {@link TextVertexReaderFromEachLineProcessedHandlingExceptions}.
*/
- protected abstract class TextVertexReader implements
- VertexReader<I, V, E> {
+ protected abstract class TextVertexReader extends VertexReader<I, V, E> {
/** Internal line record reader */
private RecordReader<LongWritable, Text> lineRecordReader;
/** Context passed to initialize */
private TaskAttemptContext context;
- /** Cached configuration */
- private ImmutableClassesGiraphConfiguration<I, V, E, Writable> conf;
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
throws IOException, InterruptedException {
this.context = context;
- conf = new ImmutableClassesGiraphConfiguration<I, V, E, Writable>(
- context.getConfiguration());
lineRecordReader = createLineRecordReader(inputSplit, context);
lineRecordReader.initialize(inputSplit, context);
}
@@ -151,15 +144,6 @@ public abstract class TextVertexInputFormat<I extends WritableComparable,
protected TaskAttemptContext getContext() {
return context;
}
-
- /**
- * Get the configuration.
- *
- * @return Configuration for this reader
- */
- protected ImmutableClassesGiraphConfiguration<I, V, E, Writable> getConf() {
- return conf;
- }
}
/**
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexOutputFormat.java b/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexOutputFormat.java
index ad96cfe..a3073f9 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexOutputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/formats/TextVertexOutputFormat.java
@@ -18,6 +18,7 @@
package org.apache.giraph.io.formats;
+import java.io.IOException;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexOutputFormat;
import org.apache.giraph.io.VertexWriter;
@@ -30,8 +31,6 @@ import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import java.io.IOException;
-
/**
* Abstract class that users should subclass to use their own text based
* vertex output format.
@@ -79,7 +78,8 @@ public abstract class TextVertexOutputFormat<I extends WritableComparable,
* vertex output. Easiest to ignore the key value separator and only use
* key instead.
*/
- protected abstract class TextVertexWriter implements VertexWriter<I, V, E> {
+ protected abstract class TextVertexWriter
+ extends VertexWriter<I, V, E> {
/** Internal line record writer */
private RecordWriter<Text, Text> lineRecordWriter;
/** Context passed to initialize */
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/iterables/EdgeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/iterables/EdgeReaderWrapper.java b/giraph-core/src/main/java/org/apache/giraph/io/iterables/EdgeReaderWrapper.java
index 4af221a..6d09d15 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/iterables/EdgeReaderWrapper.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/iterables/EdgeReaderWrapper.java
@@ -18,6 +18,7 @@
package org.apache.giraph.io.iterables;
+import java.io.IOException;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.io.EdgeReader;
import org.apache.hadoop.io.Writable;
@@ -25,8 +26,6 @@ import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/**
* Wraps {@link GiraphReader} for edges into {@link EdgeReader}
*
@@ -34,7 +33,7 @@ import java.io.IOException;
* @param <E> Edge data
*/
public class EdgeReaderWrapper<I extends WritableComparable,
- E extends Writable> implements EdgeReader<I, E> {
+ E extends Writable> extends EdgeReader<I, E> {
/** Wrapped edge reader */
private GiraphReader<EdgeWithSource<I, E>> edgeReader;
/** {@link EdgeReader}-like wrapper of {@link #edgeReader} */
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/iterables/VertexReaderWrapper.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/iterables/VertexReaderWrapper.java b/giraph-core/src/main/java/org/apache/giraph/io/iterables/VertexReaderWrapper.java
index 7e695ac..927ec3f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/iterables/VertexReaderWrapper.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/iterables/VertexReaderWrapper.java
@@ -18,6 +18,7 @@
package org.apache.giraph.io.iterables;
+import java.io.IOException;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.io.VertexReader;
import org.apache.hadoop.io.Writable;
@@ -25,8 +26,6 @@ import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.IOException;
-
/**
* Wraps {@link GiraphReader} for vertices into {@link VertexReader}
*
@@ -35,7 +34,7 @@ import java.io.IOException;
* @param <E> Edge data
*/
public class VertexReaderWrapper<I extends WritableComparable,
- V extends Writable, E extends Writable> implements VertexReader<I, V, E> {
+ V extends Writable, E extends Writable> extends VertexReader<I, V, E> {
/** Wrapped edge reader */
private GiraphReader<Vertex<I, V, E, ?>> vertexReader;
/** {@link VertexReader}-like wrapper of {@link #vertexReader} */
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/MultiThreadedSuperstepOutput.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/MultiThreadedSuperstepOutput.java b/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/MultiThreadedSuperstepOutput.java
index a09f915..af086e1 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/MultiThreadedSuperstepOutput.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/MultiThreadedSuperstepOutput.java
@@ -46,6 +46,8 @@ public class MultiThreadedSuperstepOutput<I extends WritableComparable,
SuperstepOutput<I, V, E> {
/** Mapper context */
private final Mapper<?, ?, ?, ?>.Context context;
+ /** Configuration */
+ private ImmutableClassesGiraphConfiguration<I, V, E, ?> configuration;
/** Vertex output format, used to get new vertex writers */
private final VertexOutputFormat<I, V, E> vertexOutputFormat;
/**
@@ -65,6 +67,7 @@ public class MultiThreadedSuperstepOutput<I extends WritableComparable,
public MultiThreadedSuperstepOutput(
ImmutableClassesGiraphConfiguration<I, V, E, ?> conf,
Mapper<?, ?, ?, ?>.Context context) {
+ this.configuration = conf;
vertexOutputFormat = conf.createVertexOutputFormat();
this.context = context;
availableVertexWriters = Lists.newArrayList();
@@ -77,6 +80,9 @@ public class MultiThreadedSuperstepOutput<I extends WritableComparable,
if (availableVertexWriters.isEmpty()) {
try {
vertexWriter = vertexOutputFormat.createVertexWriter(context);
+ vertexWriter.setConf(
+ (ImmutableClassesGiraphConfiguration<I, V, E, Writable>)
+ configuration);
vertexWriter.initialize(context);
} catch (IOException e) {
throw new IllegalStateException("getVertexWriter: " +
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/SynchronizedSuperstepOutput.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/SynchronizedSuperstepOutput.java b/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/SynchronizedSuperstepOutput.java
index 9617b24..2a7af29 100644
--- a/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/SynchronizedSuperstepOutput.java
+++ b/giraph-core/src/main/java/org/apache/giraph/io/superstep_output/SynchronizedSuperstepOutput.java
@@ -55,6 +55,7 @@ public class SynchronizedSuperstepOutput<I extends WritableComparable,
* @param conf Configuration
* @param context Mapper context
*/
+ @SuppressWarnings("unchecked")
public SynchronizedSuperstepOutput(
ImmutableClassesGiraphConfiguration<I, V, E, ?> conf,
Mapper<?, ?, ?, ?>.Context context) {
@@ -62,6 +63,8 @@ public class SynchronizedSuperstepOutput<I extends WritableComparable,
try {
vertexWriter =
conf.createVertexOutputFormat().createVertexWriter(context);
+ vertexWriter.setConf(
+ (ImmutableClassesGiraphConfiguration<I, V, E, Writable>) conf);
vertexWriter.initialize(context);
} catch (IOException e) {
throw new IllegalStateException("SynchronizedSuperstepOutput: " +
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/utils/InMemoryVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/InMemoryVertexInputFormat.java b/giraph-core/src/main/java/org/apache/giraph/utils/InMemoryVertexInputFormat.java
index 8e56eac..65d99db 100644
--- a/giraph-core/src/main/java/org/apache/giraph/utils/InMemoryVertexInputFormat.java
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/InMemoryVertexInputFormat.java
@@ -40,8 +40,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
* @param <E> The edge type
*/
public class InMemoryVertexInputFormat<I extends WritableComparable,
- V extends Writable,
- E extends Writable>
+ V extends Writable, E extends Writable>
extends VertexInputFormat<I, V, E> {
/** The graph */
private static TestGraph GRAPH;
@@ -67,42 +66,47 @@ public class InMemoryVertexInputFormat<I extends WritableComparable,
}
@Override
- public VertexReader<I, V, E> createVertexReader(InputSplit inputSpplit,
+ public VertexReader<I, V, E> createVertexReader(InputSplit inputSplit,
TaskAttemptContext context) throws IOException {
+ return new InMemoryVertexReader();
+ }
- return new VertexReader<I, V, E>() {
- /** The iterator */
- private Iterator<Vertex<I, V, E, ?>> vertexIterator;
- private Vertex<I, V, E, ?> currentVertex;
+ /**
+ * Simple in memory reader
+ */
+ private class InMemoryVertexReader extends VertexReader<I, V, E> {
+ /** The iterator */
+ private Iterator<Vertex<I, V, E, ?>> vertexIterator;
+ /** Current vertex */
+ private Vertex<I, V, E, ?> currentVertex;
- @Override
- public void initialize(InputSplit inputSplit,
- TaskAttemptContext context) {
- vertexIterator = GRAPH.iterator();
- }
+ @Override
+ public void initialize(InputSplit inputSplit,
+ TaskAttemptContext context) {
+ vertexIterator = GRAPH.iterator();
+ }
- @Override
- public boolean nextVertex() {
- if (vertexIterator.hasNext()) {
- currentVertex = vertexIterator.next();
- return true;
- }
- return false;
+ @Override
+ public boolean nextVertex() {
+ if (vertexIterator.hasNext()) {
+ currentVertex = vertexIterator.next();
+ return true;
}
+ return false;
+ }
- @Override
- public Vertex<I, V, E, ?> getCurrentVertex() {
- return currentVertex;
- }
+ @Override
+ public Vertex<I, V, E, ?> getCurrentVertex() {
+ return currentVertex;
+ }
- @Override
- public void close() {
- }
+ @Override
+ public void close() {
+ }
- @Override
- public float getProgress() {
- return 0;
- }
- };
+ @Override
+ public float getProgress() {
+ return 0;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java b/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
index 2ea91b5..061df68 100644
--- a/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
+++ b/giraph-core/src/main/java/org/apache/giraph/worker/BspServiceWorker.java
@@ -948,6 +948,9 @@ else[HADOOP_NON_SECURE]*/
getConfiguration().createVertexOutputFormat();
VertexWriter<I, V, E> vertexWriter =
vertexOutputFormat.createVertexWriter(getContext());
+ vertexWriter.setConf(
+ (ImmutableClassesGiraphConfiguration<I, V, E, Writable>)
+ getConfiguration());
vertexWriter.initialize(getContext());
long verticesWritten = 0;
long nextPrintVertices = 0;
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/worker/EdgeInputSplitsCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/EdgeInputSplitsCallable.java b/giraph-core/src/main/java/org/apache/giraph/worker/EdgeInputSplitsCallable.java
index 017257a..afb636b 100644
--- a/giraph-core/src/main/java/org/apache/giraph/worker/EdgeInputSplitsCallable.java
+++ b/giraph-core/src/main/java/org/apache/giraph/worker/EdgeInputSplitsCallable.java
@@ -109,6 +109,9 @@ public class EdgeInputSplitsCallable<I extends WritableComparable,
configuration.createEdgeInputFormat();
EdgeReader<I, E> edgeReader =
edgeInputFormat.createEdgeReader(inputSplit, context);
+ edgeReader.setConf(
+ (ImmutableClassesGiraphConfiguration<I, Writable, E, Writable>)
+ configuration);
edgeReader.initialize(inputSplit, context);
long inputSplitEdgesLoaded = 0;
while (edgeReader.nextEdge()) {
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/main/java/org/apache/giraph/worker/VertexInputSplitsCallable.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/worker/VertexInputSplitsCallable.java b/giraph-core/src/main/java/org/apache/giraph/worker/VertexInputSplitsCallable.java
index c18648b..c426032 100644
--- a/giraph-core/src/main/java/org/apache/giraph/worker/VertexInputSplitsCallable.java
+++ b/giraph-core/src/main/java/org/apache/giraph/worker/VertexInputSplitsCallable.java
@@ -115,6 +115,8 @@ public class VertexInputSplitsCallable<I extends WritableComparable,
configuration.createVertexInputFormat();
VertexReader<I, V, E> vertexReader =
vertexInputFormat.createVertexReader(inputSplit, context);
+ vertexReader.setConf(
+ (ImmutableClassesGiraphConfiguration<I, V, E, Writable>) configuration);
vertexReader.initialize(inputSplit, context);
long inputSplitVerticesLoaded = 0;
long edgesSinceLastUpdate = 0;
http://git-wip-us.apache.org/repos/asf/giraph/blob/84cdd9f8/giraph-core/src/test/java/org/apache/giraph/io/TestAdjacencyListTextVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/test/java/org/apache/giraph/io/TestAdjacencyListTextVertexOutputFormat.java b/giraph-core/src/test/java/org/apache/giraph/io/TestAdjacencyListTextVertexOutputFormat.java
index 60bfac1..6849e3a 100644
--- a/giraph-core/src/test/java/org/apache/giraph/io/TestAdjacencyListTextVertexOutputFormat.java
+++ b/giraph-core/src/test/java/org/apache/giraph/io/TestAdjacencyListTextVertexOutputFormat.java
@@ -18,28 +18,49 @@
package org.apache.giraph.io;
import com.google.common.collect.Lists;
-import org.apache.giraph.io.formats.AdjacencyListTextVertexOutputFormat;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.EdgeFactory;
import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.giraph.io.formats.AdjacencyListTextVertexOutputFormat;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
public class TestAdjacencyListTextVertexOutputFormat extends AdjacencyListTextVertexOutputFormat<Text, DoubleWritable, DoubleWritable> {
+ /** Test configuration */
+ private ImmutableClassesGiraphConfiguration<Text,
+ DoubleWritable, DoubleWritable, Writable> conf;
+
+ /**
+ * Dummy class to allow ImmutableClassesGiraphConfiguration to be created.
+ */
+ public static class DummyVertex extends Vertex<Text, DoubleWritable,
+ DoubleWritable, DoubleWritable> {
+ @Override
+ public void compute(Iterable<DoubleWritable> messages) throws IOException {
+ // Do nothing
+ }
+ }
+
+ @Before
+ public void setUp() {
+ GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
+ giraphConfiguration.setVertexClass(DummyVertex.class);
+ conf = new ImmutableClassesGiraphConfiguration<Text,
+ DoubleWritable, DoubleWritable, Writable>(giraphConfiguration);
+ }
protected AdjacencyListTextVertexWriter createVertexWriter(
final RecordWriter<Text, Text> tw) {
@@ -55,7 +76,6 @@ public class TestAdjacencyListTextVertexOutputFormat extends AdjacencyListTextVe
@Test
public void testVertexWithNoEdges() throws IOException, InterruptedException {
- Configuration conf = new Configuration();
TaskAttemptContext tac = mock(TaskAttemptContext.class);
when(tac.getConfiguration()).thenReturn(conf);
@@ -67,6 +87,7 @@ public class TestAdjacencyListTextVertexOutputFormat extends AdjacencyListTextVe
RecordWriter<Text, Text> tw = mock(RecordWriter.class);
AdjacencyListTextVertexWriter writer = createVertexWriter(tw);
+ writer.setConf(conf);
writer.initialize(tac);
writer.writeVertex(vertex);
@@ -77,7 +98,6 @@ public class TestAdjacencyListTextVertexOutputFormat extends AdjacencyListTextVe
@Test
public void testVertexWithEdges() throws IOException, InterruptedException {
- Configuration conf = new Configuration();
TaskAttemptContext tac = mock(TaskAttemptContext.class);
when(tac.getConfiguration()).thenReturn(conf);
@@ -94,6 +114,7 @@ public class TestAdjacencyListTextVertexOutputFormat extends AdjacencyListTextVe
RecordWriter<Text,Text> tw = mock(RecordWriter.class);
AdjacencyListTextVertexWriter writer = createVertexWriter(tw);
+ writer.setConf(conf);
writer.initialize(tac);
writer.writeVertex(vertex);
@@ -105,7 +126,6 @@ public class TestAdjacencyListTextVertexOutputFormat extends AdjacencyListTextVe
@Test
public void testWithDifferentDelimiter() throws IOException, InterruptedException {
- Configuration conf = new Configuration();
conf.set(AdjacencyListTextVertexOutputFormat.LINE_TOKENIZE_VALUE, ":::");
TaskAttemptContext tac = mock(TaskAttemptContext.class);
when(tac.getConfiguration()).thenReturn(conf);
@@ -123,6 +143,7 @@ public class TestAdjacencyListTextVertexOutputFormat extends AdjacencyListTextVe
RecordWriter<Text,Text> tw = mock(RecordWriter.class);
AdjacencyListTextVertexWriter writer = createVertexWriter(tw);
+ writer.setConf(conf);
writer.initialize(tac);
writer.writeVertex(vertex);