You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/24 22:46:14 UTC
incubator-tinkerpop git commit: added HadoopPools which holds the
GryoPool stuff globally for Spark and Giraph. VertexWritable and
ObjectWritable are now smart about serialization.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 8f5165834 -> f590faacc
added HadoopPools which holds the GryoPool stuff globally for Spark and Giraph. VertexWritable and ObjectWritable are now smart about serialization.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f590faac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f590faac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f590faac
Branch: refs/heads/master
Commit: f590faaccec68c57008450a90b2c04249afbed23
Parents: 8f51658
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 24 15:46:10 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 24 15:46:10 2015 -0600
----------------------------------------------------------------------
hadoop-gremlin/conf/hadoop-gryo.properties | 9 +++---
.../computer/giraph/GiraphWorkerContext.java | 2 --
.../hadoop/structure/io/HadoopPools.java | 32 ++++++++++++++++++++
.../hadoop/structure/io/ObjectWritable.java | 18 ++++++++---
.../hadoop/structure/io/VertexWritable.java | 5 ++-
5 files changed, 53 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f590faac/hadoop-gremlin/conf/hadoop-gryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-gryo.properties b/hadoop-gremlin/conf/hadoop-gryo.properties
index 8d9de43..6a16aed 100644
--- a/hadoop-gremlin/conf/hadoop-gryo.properties
+++ b/hadoop-gremlin/conf/hadoop-gryo.properties
@@ -21,7 +21,7 @@ gremlin.hadoop.memoryOutputFormat=org.apache.hadoop.mapreduce.lib.output.Sequenc
gremlin.hadoop.deriveMemory=false
gremlin.hadoop.jarsInDistributedCache=true
-gremlin.hadoop.inputLocation=tinkerpop-modern-vertices.kryo
+gremlin.hadoop.inputLocation=100K.gio
gremlin.hadoop.outputLocation=output
#####################################
@@ -45,6 +45,7 @@ giraph.maxMessagesInMemory=100000
# SparkGraphComputer Configuration #
####################################
spark.master=local[4]
-spark.executor.memory=1024m
-spark.eventLog.enabled=true
-spark.serializer=org.apache.spark.serializer.JavaSerializer
\ No newline at end of file
+spark.executor.memory=1g
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+# spark.eventLog.enabled=true
+# spark.eventLog.dir=/tmp/spark-event-logs
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f590faac/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
index dc4d51a..c459c1d 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
@@ -33,8 +33,6 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
*/
public final class GiraphWorkerContext extends WorkerContext {
- public static final GryoPool GRYO_POOL = new GryoPool(256);
-
private VertexProgramPool vertexProgramPool;
private GiraphMemory memory;
private boolean deriveMemory;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f590faac/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
new file mode 100644
index 0000000..725b588
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class HadoopPools {
+
+ private HadoopPools() {
+ }
+
+ public static final GryoPool GRYO_POOL = new GryoPool(256);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f590faac/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
index 9cc942c..72833f6 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphWorkerContext;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoInput;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoOutput;
@@ -29,11 +28,14 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class ObjectWritable<T> implements WritableComparable<ObjectWritable> {
+public final class ObjectWritable<T> implements WritableComparable<ObjectWritable>, Serializable {
T t;
@@ -59,7 +61,7 @@ public final class ObjectWritable<T> implements WritableComparable<ObjectWritabl
@Override
public void readFields(final DataInput input) throws IOException {
- this.t = GiraphWorkerContext.GRYO_POOL.doWithReader(gryoReader -> {
+ this.t = HadoopPools.GRYO_POOL.doWithReader(gryoReader -> {
try {
return gryoReader.readObject(new GryoInput(new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input))));
} catch (IOException e) {
@@ -70,7 +72,7 @@ public final class ObjectWritable<T> implements WritableComparable<ObjectWritabl
@Override
public void write(final DataOutput output) throws IOException {
- GiraphWorkerContext.GRYO_POOL.doWithWriter(gryoWriter -> {
+ HadoopPools.GRYO_POOL.doWithWriter(gryoWriter -> {
try {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final GryoOutput out = new GryoOutput(outputStream);
@@ -84,6 +86,14 @@ public final class ObjectWritable<T> implements WritableComparable<ObjectWritabl
});
}
+ private void writeObject(final ObjectOutputStream outputStream) throws IOException {
+ this.write(outputStream);
+ }
+
+ private void readObject(final ObjectInputStream inputStream) throws IOException, ClassNotFoundException {
+ this.readFields(inputStream);
+ }
+
@Override
public int compareTo(final ObjectWritable objectWritable) {
return this.t instanceof Comparable ? ((Comparable) this.t).compareTo(objectWritable.get()) : 0;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f590faac/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
index 92d10b7..9608e7d 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphWorkerContext;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -66,7 +65,7 @@ public final class VertexWritable implements Writable, Serializable {
public void readFields(final DataInput input) throws IOException {
try {
this.vertex = null;
- this.vertex = GiraphWorkerContext.GRYO_POOL.doWithReader(gryoReader -> {
+ this.vertex = HadoopPools.GRYO_POOL.doWithReader(gryoReader -> {
try {
final ByteArrayInputStream inputStream = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
final Graph gLocal = TinkerGraph.open();
@@ -88,7 +87,7 @@ public final class VertexWritable implements Writable, Serializable {
@Override
public void write(final DataOutput output) throws IOException {
try {
- GiraphWorkerContext.GRYO_POOL.doWithWriter(gryoWriter -> {
+ HadoopPools.GRYO_POOL.doWithWriter(gryoWriter -> {
try {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
gryoWriter.writeVertex(outputStream, this.vertex, Direction.BOTH);