You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by jg...@apache.org on 2011/11/17 22:37:05 UTC
svn commit: r1203379 - in /incubator/giraph/trunk: ./
src/main/java/org/apache/giraph/lib/ src/test/java/org/apache/giraph/lib/
Author: jghoman
Date: Thu Nov 17 21:37:05 2011
New Revision: 1203379
URL: http://svn.apache.org/viewvc?rev=1203379&view=rev
Log:
GIRAPH-92. Need outputformat for just vertex ID and value.
Added:
incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java
incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java
Modified:
incubator/giraph/trunk/CHANGELOG
incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java
Modified: incubator/giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/CHANGELOG?rev=1203379&r1=1203378&r2=1203379&view=diff
==============================================================================
--- incubator/giraph/trunk/CHANGELOG (original)
+++ incubator/giraph/trunk/CHANGELOG Thu Nov 17 21:37:05 2011
@@ -2,6 +2,8 @@ Giraph Change Log
Release 0.70.0 - unreleased
+ GIRAPH-92: Need outputformat for just vertex ID and value. (jghoman)
+
GIRAPH-86: Simplify boolean expressions in ZooKeeperExt::createExt.
(attilacsordas via jghoman)
Modified: incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java?rev=1203379&r1=1203378&r2=1203379&view=diff
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java (original)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/AdjacencyListTextVertexOutputFormat.java Thu Nov 17 21:37:05 2011
@@ -43,7 +43,7 @@ public class AdjacencyListTextVertexOutp
static class AdjacencyListVertexWriter<I extends WritableComparable, V extends
Writable, E extends Writable> extends TextVertexWriter<I, V, E> {
- public static final String LINE_TOKENIZE_VALUE = "adj.list.output.delimiter";
+ public static final String LINE_TOKENIZE_VALUE = "output.delimiter";
public static final String LINE_TOKENIZE_VALUE_DEFAULT = "\t";
private String delimiter;
Added: incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java?rev=1203379&view=auto
==============================================================================
--- incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java (added)
+++ incubator/giraph/trunk/src/main/java/org/apache/giraph/lib/IdWithValueTextOutputFormat.java Thu Nov 17 21:37:05 2011
@@ -0,0 +1,76 @@
+package org.apache.giraph.lib;
+
+
+import org.apache.giraph.graph.BasicVertex;
+import org.apache.giraph.graph.VertexWriter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+
+/**
+ * Write out Vertices' IDs and values, but not their edges nor edges' values.
+ * This is a useful output format when the final value of the vertex is
+ * all that's needed. The boolean configuration parameter reverse.id.and.value
+ * allows reversing the output of id and value.
+ *
+ * @param <I> Vertex index value
+ * @param <V> Vertex value
+ * @param <E> Edge value
+ */
+public class IdWithValueTextOutputFormat <I extends WritableComparable,
+ V extends Writable, E extends Writable> extends TextVertexOutputFormat<I, V, E>{
+
+ static class IdWithValueVertexWriter<I extends WritableComparable, V extends
+ Writable, E extends Writable> extends TextVertexWriter<I, V, E> {
+
+ public static final String LINE_TOKENIZE_VALUE = "output.delimiter";
+ public static final String LINE_TOKENIZE_VALUE_DEFAULT = "\t";
+
+ public static final String REVERSE_ID_AND_VALUE = "reverse.id.and.value";
+ public static final boolean REVERSE_ID_AND_VALUE_DEFAULT = false;
+
+ private String delimiter;
+
+ public IdWithValueVertexWriter(RecordWriter<Text, Text> recordWriter) {
+ super(recordWriter);
+ }
+
+ @Override
+ public void writeVertex(BasicVertex<I, V, E, ?> vertex) throws IOException,
+ InterruptedException {
+ if (delimiter == null) {
+ delimiter = getContext().getConfiguration()
+ .get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT);
+ }
+
+ String first;
+ String second;
+ boolean reverseOutput = getContext().getConfiguration()
+ .getBoolean(REVERSE_ID_AND_VALUE, REVERSE_ID_AND_VALUE_DEFAULT);
+
+ if (reverseOutput) {
+ first = vertex.getVertexValue().toString();
+ second = vertex.getVertexId().toString();
+ } else {
+ first = vertex.getVertexId().toString();
+ second = vertex.getVertexValue().toString();
+ }
+
+ Text line = new Text(first + delimiter + second);
+
+ getRecordWriter().write(line, null);
+ }
+ }
+
+ @Override
+ public VertexWriter<I, V, E> createVertexWriter(TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ return new IdWithValueVertexWriter<I, V, E>
+ (textOutputFormat.getRecordWriter(context));
+ }
+
+}
Added: incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java?rev=1203379&view=auto
==============================================================================
--- incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java (added)
+++ incubator/giraph/trunk/src/test/java/org/apache/giraph/lib/TestIdWithValueTextOutputFormat.java Thu Nov 17 21:37:05 2011
@@ -0,0 +1,70 @@
+package org.apache.giraph.lib;
+
+import junit.framework.TestCase;
+import org.apache.giraph.graph.BasicVertex;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.mockito.Matchers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter;
+import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter.LINE_TOKENIZE_VALUE;
+import static org.apache.giraph.lib.IdWithValueTextOutputFormat.IdWithValueVertexWriter.REVERSE_ID_AND_VALUE;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestIdWithValueTextOutputFormat extends TestCase {
+ public void testHappyPath() throws IOException, InterruptedException {
+ Configuration conf = new Configuration();
+ Text expected = new Text("Four Tops\t4.0");
+
+ IdWithValueTestWorker(conf, expected);
+ }
+
+ public void testReverseIdAndValue() throws IOException, InterruptedException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(REVERSE_ID_AND_VALUE, true);
+ Text expected = new Text("4.0\tFour Tops");
+
+ IdWithValueTestWorker(conf, expected);
+ }
+
+ public void testWithDifferentDelimiter() throws IOException,
+ InterruptedException {
+ Configuration conf = new Configuration();
+ conf.set(LINE_TOKENIZE_VALUE, "blah");
+ Text expected = new Text("Four Topsblah4.0");
+
+ IdWithValueTestWorker(conf, expected);
+ }
+
+ private void IdWithValueTestWorker(Configuration conf, Text expected)
+ throws IOException, InterruptedException {
+ TaskAttemptContext tac = mock(TaskAttemptContext.class);
+ when(tac.getConfiguration()).thenReturn(conf);
+
+ BasicVertex vertex = mock(BasicVertex.class);
+ when(vertex.getVertexId()).thenReturn(new Text("Four Tops"));
+ when(vertex.getVertexValue()).thenReturn(new DoubleWritable(4d));
+
+ // Create empty iterator == no edges
+ when(vertex.iterator()).thenReturn(new ArrayList<Text>().iterator());
+
+ RecordWriter<Text, Text> tw = mock(RecordWriter.class);
+ IdWithValueVertexWriter writer = new IdWithValueVertexWriter(tw);
+ writer.initialize(tac);
+ writer.writeVertex(vertex);
+
+ verify(tw).write(expected, null);
+ verify(vertex, times(0)).iterator();
+ verify(vertex, times(0)).getEdgeValue(Matchers.<WritableComparable>any());
+ }
+}