You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ma...@apache.org on 2013/10/18 03:32:01 UTC
git commit: updated refs/heads/trunk to fad298e
Updated Branches:
refs/heads/trunk b3eada3c0 -> fad298ebb
GIRAPH-786: XSparseVector create a lot of objects in add/write (ikabiljo via majakabiljo)
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/fad298eb
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/fad298eb
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/fad298eb
Branch: refs/heads/trunk
Commit: fad298ebb5a35c7466aac69af57f60775ff6a87f
Parents: b3eada3
Author: Maja Kabiljo <ma...@fb.com>
Authored: Thu Oct 17 18:31:31 2013 -0700
Committer: Maja Kabiljo <ma...@fb.com>
Committed: Thu Oct 17 18:31:31 2013 -0700
----------------------------------------------------------------------
CHANGELOG | 3 +++
.../matrix/sparse/DoubleSparseVector.java | 12 ++++++++++--
.../matrix/sparse/FloatSparseVector.java | 12 ++++++++++--
.../matrix/sparse/IntSparseVector.java | 12 ++++++++++--
.../matrix/sparse/LongSparseVector.java | 18 +++++++++++++-----
5 files changed, 46 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index f457026..9d55b47 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
Giraph Change Log
Release 1.1.0 - unreleased
+ GIRAPH-786: XSparseVector create a lot of objects in add/write
+ (ikabiljo via majakabiljo)
+
GIRAPH-773: Cannot use only EdgeOutputFormat (armax00 via claudio)
GIRAPH-779: Make it easier to reuse objects with hive-io input (majakabiljo)
http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java
index de119fb..fb54459 100644
--- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/DoubleSparseVector.java
@@ -19,11 +19,13 @@
package org.apache.giraph.aggregators.matrix.sparse;
import it.unimi.dsi.fastutil.ints.Int2DoubleMap;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
+import it.unimi.dsi.fastutil.objects.ObjectIterator;
import org.apache.hadoop.io.Writable;
@@ -96,7 +98,10 @@ public class DoubleSparseVector implements Writable {
* @param other the vector to add.
*/
public void add(DoubleSparseVector other) {
- for (Int2DoubleMap.Entry entry : other.entries.int2DoubleEntrySet()) {
+ ObjectIterator<Int2DoubleMap.Entry> iter =
+ other.entries.int2DoubleEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2DoubleMap.Entry entry = iter.next();
entries.addTo(entry.getIntKey(), entry.getDoubleValue());
}
}
@@ -104,7 +109,10 @@ public class DoubleSparseVector implements Writable {
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(entries.size());
- for (Int2DoubleMap.Entry entry : entries.int2DoubleEntrySet()) {
+ ObjectIterator<Int2DoubleMap.Entry> iter =
+ entries.int2DoubleEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2DoubleMap.Entry entry = iter.next();
out.writeInt(entry.getIntKey());
out.writeDouble(entry.getDoubleValue());
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java
index 4e94217..77a9bd9 100644
--- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/FloatSparseVector.java
@@ -19,11 +19,13 @@
package org.apache.giraph.aggregators.matrix.sparse;
import it.unimi.dsi.fastutil.ints.Int2FloatMap;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import it.unimi.dsi.fastutil.ints.Int2FloatOpenHashMap;
+import it.unimi.dsi.fastutil.objects.ObjectIterator;
import org.apache.hadoop.io.Writable;
@@ -96,7 +98,10 @@ public class FloatSparseVector implements Writable {
* @param other the vector to add.
*/
public void add(FloatSparseVector other) {
- for (Int2FloatMap.Entry entry : other.entries.int2FloatEntrySet()) {
+ ObjectIterator<Int2FloatMap.Entry> iter =
+ other.entries.int2FloatEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2FloatMap.Entry entry = iter.next();
entries.addTo(entry.getIntKey(), entry.getFloatValue());
}
}
@@ -104,7 +109,10 @@ public class FloatSparseVector implements Writable {
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(entries.size());
- for (Int2FloatMap.Entry entry : entries.int2FloatEntrySet()) {
+ ObjectIterator<Int2FloatMap.Entry> iter =
+ entries.int2FloatEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2FloatMap.Entry entry = iter.next();
out.writeInt(entry.getIntKey());
out.writeFloat(entry.getFloatValue());
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java
index 2482bd2..0ed6bc3 100644
--- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/IntSparseVector.java
@@ -19,11 +19,13 @@
package org.apache.giraph.aggregators.matrix.sparse;
import it.unimi.dsi.fastutil.ints.Int2IntMap;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
+import it.unimi.dsi.fastutil.objects.ObjectIterator;
import org.apache.hadoop.io.Writable;
@@ -96,7 +98,10 @@ public class IntSparseVector implements Writable {
* @param other the vector to add.
*/
public void add(IntSparseVector other) {
- for (Int2IntMap.Entry entry : other.entries.int2IntEntrySet()) {
+ ObjectIterator<Int2IntMap.Entry> iter =
+ other.entries.int2IntEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2IntMap.Entry entry = iter.next();
entries.addTo(entry.getIntKey(), entry.getIntValue());
}
}
@@ -104,7 +109,10 @@ public class IntSparseVector implements Writable {
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(entries.size());
- for (Int2IntMap.Entry entry : entries.int2IntEntrySet()) {
+ ObjectIterator<Int2IntMap.Entry> iter =
+ entries.int2IntEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2IntMap.Entry entry = iter.next();
out.writeInt(entry.getIntKey());
out.writeInt(entry.getIntValue());
}
http://git-wip-us.apache.org/repos/asf/giraph/blob/fad298eb/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java
index 8ab52a9..6337215 100644
--- a/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java
+++ b/giraph-core/src/main/java/org/apache/giraph/aggregators/matrix/sparse/LongSparseVector.java
@@ -19,11 +19,13 @@
package org.apache.giraph.aggregators.matrix.sparse;
import it.unimi.dsi.fastutil.ints.Int2LongMap;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap;
+import it.unimi.dsi.fastutil.objects.ObjectIterator;
import org.apache.hadoop.io.Writable;
@@ -96,17 +98,23 @@ public class LongSparseVector implements Writable {
* @param other the vector to add.
*/
public void add(LongSparseVector other) {
- for (Int2LongMap.Entry kv : other.entries.int2LongEntrySet()) {
- entries.addTo(kv.getIntKey(), kv.getLongValue());
+ ObjectIterator<Int2LongMap.Entry> iter =
+ other.entries.int2LongEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2LongMap.Entry entry = iter.next();
+ entries.addTo(entry.getIntKey(), entry.getLongValue());
}
}
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(entries.size());
- for (Int2LongMap.Entry kv : entries.int2LongEntrySet()) {
- out.writeInt(kv.getIntKey());
- out.writeLong(kv.getLongValue());
+ ObjectIterator<Int2LongMap.Entry> iter =
+ entries.int2LongEntrySet().fastIterator();
+ while (iter.hasNext()) {
+ Int2LongMap.Entry entry = iter.next();
+ out.writeInt(entry.getIntKey());
+ out.writeLong(entry.getLongValue());
}
}