You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ma...@apache.org on 2013/04/18 22:03:20 UTC
git commit: updated refs/heads/trunk to 75b3112
Updated Branches:
refs/heads/trunk 32b7f593f -> 75b311266
GIRAPH-636: Initialize compute OutEdges directly from input OutEdges (majakabiljo)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/75b31126
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/75b31126
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/75b31126
Branch: refs/heads/trunk
Commit: 75b3112661c0900d7447d8ad56647b3e73ec0a24
Parents: 32b7f59
Author: Maja Kabiljo <ma...@maja-mbp.thefacebook.com>
Authored: Thu Apr 18 13:01:14 2013 -0700
Committer: Maja Kabiljo <ma...@maja-mbp.thefacebook.com>
Committed: Thu Apr 18 13:02:34 2013 -0700
----------------------------------------------------------------------
CHANGELOG | 3 +
.../java/org/apache/giraph/edge/EdgeStore.java | 7 +--
.../java/org/apache/giraph/edge/HashMapEdges.java | 14 +----
.../org/apache/giraph/edge/HashMultimapEdges.java | 14 +----
.../org/apache/giraph/edge/IntNullArrayEdges.java | 15 +----
.../apache/giraph/edge/LongDoubleArrayEdges.java | 16 +----
.../apache/giraph/edge/LongDoubleHashMapEdges.java | 15 +----
.../org/apache/giraph/edge/LongNullArrayEdges.java | 16 +----
.../apache/giraph/edge/LongNullHashSetEdges.java | 15 +----
.../org/apache/giraph/utils/EdgeIterables.java | 47 +++++++++++++++
10 files changed, 71 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index c1a2c85..d7679a2 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
Giraph Change Log
Release 1.0.1 - unreleased
+ GIRAPH-636: Initialize compute OutEdges directly from input OutEdges
+ (majakabiljo)
+
GIRAPH-645: Specify maven deploy plugin version (nitay)
GIRAPH-643: Maven Release settings more friendly to Git (nitay)
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java b/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
index 81044bd..9e2d246 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/EdgeStore.java
@@ -146,12 +146,7 @@ public class EdgeStore<I extends WritableComparable,
if (!useInputOutEdges) {
return inputEdges;
} else {
- OutEdges<I, E> computeEdges =
- configuration.createAndInitializeOutEdges(inputEdges.size());
- for (Edge<I, E> edge : inputEdges) {
- computeEdges.add(edge);
- }
- return computeEdges;
+ return configuration.createAndInitializeOutEdges(inputEdges);
}
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java
index fc23cf0..0945634 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/HashMapEdges.java
@@ -19,13 +19,14 @@
package org.apache.giraph.edge;
import com.google.common.collect.Maps;
+
+import org.apache.giraph.utils.EdgeIterables;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -48,16 +49,7 @@ public class HashMapEdges<I extends WritableComparable, E extends Writable>
@Override
public void initialize(Iterable<Edge<I, E>> edges) {
- // If the iterable is actually a collection, we can cheaply get the
- // size and initialize the hash-map with the expected capacity.
- if (edges instanceof Collection) {
- initialize(((Collection<Edge<I, E>>) edges).size());
- } else {
- initialize();
- }
- for (Edge<I, E> edge : edges) {
- add(edge);
- }
+ EdgeIterables.initialize(this, edges);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java
index abe6151..e5e39de 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/HashMultimapEdges.java
@@ -20,13 +20,14 @@ package org.apache.giraph.edge;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.UnmodifiableIterator;
+
+import org.apache.giraph.utils.EdgeIterables;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
@@ -47,16 +48,7 @@ public class HashMultimapEdges<I extends WritableComparable, E extends Writable>
@Override
public void initialize(Iterable<Edge<I, E>> edges) {
- // If the iterable is actually a collection, we can cheaply get the
- // size and initialize the hash-multimap with the expected capacity.
- if (edges instanceof Collection) {
- initialize(((Collection<Edge<I, E>>) edges).size());
- } else {
- initialize();
- }
- for (Edge<I, E> edge : edges) {
- add(edge);
- }
+ EdgeIterables.initialize(this, edges);
}
/**
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java
index 9867771..e54b41f 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/IntNullArrayEdges.java
@@ -18,6 +18,7 @@
package org.apache.giraph.edge;
+import org.apache.giraph.utils.EdgeIterables;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
@@ -29,7 +30,6 @@ import it.unimi.dsi.fastutil.ints.IntIterator;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Collection;
import java.util.Iterator;
/**
@@ -46,18 +46,7 @@ public class IntNullArrayEdges
@Override
public void initialize(Iterable<Edge<IntWritable, NullWritable>> edges) {
- // If the iterable is actually a collection, we can cheaply get the
- // size and initialize the array with the expected capacity.
- if (edges instanceof Collection) {
- int numEdges =
- ((Collection<Edge<IntWritable, NullWritable>>) edges).size();
- initialize(numEdges);
- } else {
- initialize();
- }
- for (Edge<IntWritable, NullWritable> edge : edges) {
- add(edge);
- }
+ EdgeIterables.initialize(this, edges);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java
index 45d182e..324a32c 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleArrayEdges.java
@@ -23,13 +23,14 @@ import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
import it.unimi.dsi.fastutil.doubles.DoubleIterator;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongIterator;
+
+import org.apache.giraph.utils.EdgeIterables;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Collection;
import java.util.Iterator;
/**
@@ -49,18 +50,7 @@ public class LongDoubleArrayEdges
@Override
public void initialize(Iterable<Edge<LongWritable, DoubleWritable>> edges) {
- // If the iterable is actually a collection, we can cheaply get the
- // size and initialize the arrays with the expected capacity.
- if (edges instanceof Collection) {
- int numEdges =
- ((Collection<Edge<LongWritable, DoubleWritable>>) edges).size();
- initialize(numEdges);
- } else {
- initialize();
- }
- for (Edge<LongWritable, DoubleWritable> edge : edges) {
- add(edge);
- }
+ EdgeIterables.initialize(this, edges);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java
index c33200d..e6c8229 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongDoubleHashMapEdges.java
@@ -22,13 +22,14 @@ import com.google.common.collect.UnmodifiableIterator;
import it.unimi.dsi.fastutil.longs.Long2DoubleMap;
import it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
+
+import org.apache.giraph.utils.EdgeIterables;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Collection;
import java.util.Iterator;
/**
@@ -50,17 +51,7 @@ public class LongDoubleHashMapEdges
@Override
public void initialize(Iterable<Edge<LongWritable, DoubleWritable>> edges) {
- // If the iterable is actually a collection, we can cheaply get the
- // size and initialize the hash-map with the expected capacity.
- if (edges instanceof Collection) {
- initialize(
- ((Collection<Edge<LongWritable, DoubleWritable>>) edges).size());
- } else {
- initialize();
- }
- for (Edge<LongWritable, DoubleWritable> edge : edges) {
- add(edge);
- }
+ EdgeIterables.initialize(this, edges);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java
index d753588..b7b3af5 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullArrayEdges.java
@@ -20,13 +20,14 @@ package org.apache.giraph.edge;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import it.unimi.dsi.fastutil.longs.LongIterator;
+
+import org.apache.giraph.utils.EdgeIterables;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Collection;
import java.util.Iterator;
/**
@@ -44,18 +45,7 @@ public class LongNullArrayEdges
@Override
public void initialize(Iterable<Edge<LongWritable, NullWritable>> edges) {
- // If the iterable is actually a collection, we can cheaply get the
- // size and initialize the arrays with the expected capacity.
- if (edges instanceof Collection) {
- int numEdges =
- ((Collection<Edge<LongWritable, NullWritable>>) edges).size();
- initialize(numEdges);
- } else {
- initialize();
- }
- for (Edge<LongWritable, NullWritable> edge : edges) {
- add(edge);
- }
+ EdgeIterables.initialize(this, edges);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java
index ef0302c..139fc7a 100644
--- a/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java
+++ b/giraph-core/src/main/java/org/apache/giraph/edge/LongNullHashSetEdges.java
@@ -20,13 +20,14 @@ package org.apache.giraph.edge;
import it.unimi.dsi.fastutil.longs.LongIterator;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
+
+import org.apache.giraph.utils.EdgeIterables;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.Collection;
import java.util.Iterator;
/**
@@ -45,17 +46,7 @@ public class LongNullHashSetEdges
@Override
public void initialize(Iterable<Edge<LongWritable, NullWritable>> edges) {
- // If the iterable is actually a collection, we can cheaply get the
- // size and initialize the hash-map with the expected capacity.
- if (edges instanceof Collection) {
- initialize(
- ((Collection<Edge<LongWritable, NullWritable>>) edges).size());
- } else {
- initialize();
- }
- for (Edge<LongWritable, NullWritable> edge : edges) {
- add(edge);
- }
+ EdgeIterables.initialize(this, edges);
}
@Override
http://git-wip-us.apache.org/repos/asf/giraph/blob/75b31126/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java b/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java
index e945d7b..0d2b810 100644
--- a/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/EdgeIterables.java
@@ -21,11 +21,14 @@ package org.apache.giraph.utils;
import com.google.common.collect.Iterables;
import org.apache.giraph.edge.Edge;
import org.apache.giraph.edge.EdgeFactory;
+import org.apache.giraph.edge.OutEdges;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
@@ -105,4 +108,48 @@ public class EdgeIterables {
Collections.sort(edgeList2, edgeComparator);
return equals(edgeList1, edgeList2);
}
+
+ /**
+ * Get the size of edges. Optimized implementation for cases when edges is
+ * instance of {@link OutEdges} or {@link Collection} which only calls
+ * size() method from these classes.
+ *
+ * @param edges Edges
+ * @param <I> Vertex index
+ * @param <E> Edge value
+ * @return Size of edges
+ */
+ public static <I extends WritableComparable, E extends Writable> int size(
+ Iterable<Edge<I, E>> edges) {
+ if (edges instanceof OutEdges) {
+ return ((OutEdges) edges).size();
+ } else {
+ return Iterables.size(edges);
+ }
+ }
+
+ /**
+ * Initialize edges data structure and add the edges from edgesIterable.
+ *
+ * If edgesIterable is instance of {@link OutEdges} or {@link Collection}
+ * edges will be initialized with size of edgesIterable,
+ * otherwise edges will be initialized without size.
+ *
+ * @param edges Edges to initialize
+ * @param edgesIterable Iterable whose edges to use
+ * @param <I> Vertex index
+ * @param <E> Edge value
+ */
+ public static <I extends WritableComparable, E extends Writable>
+ void initialize(OutEdges<I, E> edges, Iterable<Edge<I, E>> edgesIterable) {
+ if (edgesIterable instanceof OutEdges ||
+ edgesIterable instanceof Collection) {
+ edges.initialize(size(edgesIterable));
+ } else {
+ edges.initialize();
+ }
+ for (Edge<I, E> edge : edgesIterable) {
+ edges.add(edge);
+ }
+ }
}