You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/04 15:42:07 UTC
[12/20] incubator-tinkerpop git commit: fixed up spark/ branch now
that is is Gryo and .kryo. @spmallette
fixed up spark/ branch now that is is Gryo and .kryo. @spmallette
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/96ffd77c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/96ffd77c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/96ffd77c
Branch: refs/heads/master
Commit: 96ffd77cc1b33ea9b054e0a32920e7079ecdfd6a
Parents: 70fc529 929a288
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Mar 3 11:10:39 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Mar 3 11:10:39 2015 -0700
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
data/grateful-dead-vertices.gio | Bin 1028298 -> 0 bytes
data/grateful-dead-vertices.kryo | Bin 0 -> 1028298 bytes
data/grateful-dead.gio | Bin 515409 -> 0 bytes
data/grateful-dead.kryo | Bin 0 -> 515409 bytes
data/tinkerpop-classic-vertices.gio | Bin 1323 -> 0 bytes
data/tinkerpop-classic-vertices.kryo | Bin 0 -> 1323 bytes
data/tinkerpop-classic.gio | Bin 817 -> 0 bytes
data/tinkerpop-classic.kryo | Bin 0 -> 817 bytes
data/tinkerpop-crew-vertices.gio | Bin 2743 -> 0 bytes
data/tinkerpop-crew-vertices.kryo | Bin 0 -> 2743 bytes
data/tinkerpop-crew.gio | Bin 1916 -> 0 bytes
data/tinkerpop-crew.kryo | Bin 0 -> 1916 bytes
data/tinkerpop-modern-vertices.gio | Bin 1439 -> 0 bytes
data/tinkerpop-modern-vertices.kryo | Bin 0 -> 1439 bytes
data/tinkerpop-modern.gio | Bin 877 -> 0 bytes
data/tinkerpop-modern.kryo | Bin 0 -> 877 bytes
docs/src/implementations.asciidoc | 14 +-
docs/src/the-graph.asciidoc | 18 +-
gremlin-console/conf/remote-objects.yaml | 2 +-
gremlin-console/conf/remote.yaml | 2 +-
.../plugin/gremlin-server-integration.yaml | 4 +-
.../gremlin/console/groovy/plugin/remote.yaml | 2 +-
.../util/metric/DependantMutableMetrics.java | 2 +-
.../process/util/metric/MutableMetrics.java | 2 +-
.../tinkerpop/gremlin/structure/Graph.java | 50 +--
.../gremlin/structure/io/DefaultIo.java | 8 +-
.../gremlin/structure/io/GraphMigrator.java | 18 +-
.../structure/io/gryo/EdgeTerminator.java | 48 +++
.../structure/io/gryo/EntrySerializer.java | 43 ++
.../structure/io/gryo/GraphSerializer.java | 164 ++++++++
.../structure/io/gryo/GremlinClassResolver.java | 206 ++++++++++
.../gremlin/structure/io/gryo/GryoMapper.java | 406 +++++++++++++++++++
.../gremlin/structure/io/gryo/GryoReader.java | 400 ++++++++++++++++++
.../gremlin/structure/io/gryo/GryoWriter.java | 180 ++++++++
.../structure/io/gryo/URISerializer.java | 46 +++
.../structure/io/gryo/UUIDSerializer.java | 46 +++
.../io/gryo/VertexByteArrayInputStream.java | 74 ++++
.../structure/io/gryo/VertexTerminator.java | 53 +++
.../structure/io/kryo/EdgeTerminator.java | 48 ---
.../structure/io/kryo/EntrySerializer.java | 43 --
.../structure/io/kryo/GraphSerializer.java | 164 --------
.../structure/io/kryo/GremlinClassResolver.java | 206 ----------
.../gremlin/structure/io/kryo/KryoMapper.java | 406 -------------------
.../gremlin/structure/io/kryo/KryoReader.java | 400 ------------------
.../gremlin/structure/io/kryo/KryoWriter.java | 180 --------
.../structure/io/kryo/URISerializer.java | 46 ---
.../structure/io/kryo/UUIDSerializer.java | 46 ---
.../io/kryo/VertexByteArrayInputStream.java | 74 ----
.../structure/io/kryo/VertexTerminator.java | 53 ---
.../tinkerpop/gremlin/driver/Cluster.java | 2 +-
.../driver/ser/GryoMessageSerializerV1d0.java | 307 ++++++++++++++
.../driver/ser/JsonBuilderGryoSerializer.java | 45 ++
.../driver/ser/JsonBuilderKryoSerializer.java | 45 --
.../driver/ser/KryoMessageSerializerV1d0.java | 307 --------------
.../tinkerpop/gremlin/driver/ser/SerTokens.java | 2 +-
.../gremlin/driver/ser/Serializers.java | 6 +-
.../gremlin/driver/simple/NioClient.java | 4 +-
.../gremlin/driver/simple/WebSocketClient.java | 4 +-
.../ser/GryoMessageSerializerV1D0Test.java | 294 ++++++++++++++
.../ser/KryoMessageSerializerV1d0Test.java | 294 --------------
.../jsr223/GremlinGroovyScriptEngineTest.java | 81 ++--
.../AbstractImportCustomizerProvider.java | 4 +-
gremlin-server/conf/gremlin-server-classic.yaml | 4 +-
gremlin-server/conf/gremlin-server-modern.yaml | 4 +-
gremlin-server/conf/gremlin-server-neo4j.yaml | 4 +-
gremlin-server/conf/gremlin-server.yaml | 4 +-
.../server/GremlinAdditionPerformanceTest.java | 2 +-
.../server/GremlinDriverIntegrateTest.java | 10 +-
.../server/gremlin-server-integration.yaml | 4 +-
.../server/gremlin-server-performance.yaml | 2 +-
.../gremlin/AbstractGraphProvider.java | 8 +-
.../apache/tinkerpop/gremlin/LoadGraphWith.java | 10 +-
.../structure/GraphWritePerformanceTest.java | 6 +-
.../tinkerpop/gremlin/structure/IoTest.java | 180 ++++----
.../gremlin/structure/SerializationTest.java | 27 +-
.../structure/io/gryo/GryoResourceAccess.java | 27 ++
.../structure/io/kryo/KryoResourceAccess.java | 27 --
.../io/gryo/grateful-dead-vertices.kryo | Bin 0 -> 1028298 bytes
.../structure/io/gryo/grateful-dead.kryo | Bin 0 -> 515409 bytes
.../io/gryo/tinkerpop-classic-vertices.kryo | Bin 0 -> 1323 bytes
.../structure/io/gryo/tinkerpop-classic.kryo | Bin 0 -> 817 bytes
.../io/gryo/tinkerpop-crew-vertices.kryo | Bin 0 -> 2743 bytes
.../structure/io/gryo/tinkerpop-crew.kryo | Bin 0 -> 1916 bytes
.../io/gryo/tinkerpop-modern-vertices.kryo | Bin 0 -> 1439 bytes
.../structure/io/gryo/tinkerpop-modern.kryo | Bin 0 -> 877 bytes
.../io/kryo/grateful-dead-vertices.gio | Bin 1028298 -> 0 bytes
.../gremlin/structure/io/kryo/grateful-dead.gio | Bin 515409 -> 0 bytes
.../io/kryo/tinkerpop-classic-vertices.gio | Bin 1323 -> 0 bytes
.../structure/io/kryo/tinkerpop-classic.gio | Bin 817 -> 0 bytes
.../io/kryo/tinkerpop-crew-vertices.gio | Bin 2743 -> 0 bytes
.../structure/io/kryo/tinkerpop-crew.gio | Bin 1916 -> 0 bytes
.../io/kryo/tinkerpop-modern-vertices.gio | Bin 1439 -> 0 bytes
.../structure/io/kryo/tinkerpop-modern.gio | Bin 877 -> 0 bytes
.../structure/io/gryo/GryoMapperTest.java | 34 ++
.../structure/io/kryo/KryoMapperTest.java | 34 --
hadoop-gremlin/conf/giraph-gryo.properties | 31 ++
hadoop-gremlin/conf/giraph-kryo.properties | 31 --
hadoop-gremlin/conf/spark-kryo.properties | 6 +-
.../groovy/plugin/HadoopGremlinPlugin.java | 4 +-
.../computer/giraph/GiraphComputeVertex.java | 8 +-
.../process/computer/giraph/RuleWritable.java | 4 +-
.../process/computer/spark/SparkVertex.java | 12 +-
.../hadoop/structure/io/ObjectWritable.java | 4 +-
.../hadoop/structure/io/VertexWritable.java | 8 +-
.../structure/io/gryo/GryoInputFormat.java | 63 +++
.../structure/io/gryo/GryoOutputFormat.java | 43 ++
.../structure/io/gryo/GryoRecordReader.java | 113 ++++++
.../structure/io/gryo/GryoRecordWriter.java | 54 +++
.../structure/io/gryo/VertexStreamIterator.java | 147 +++++++
.../structure/io/kryo/KryoInputFormat.java | 63 ---
.../structure/io/kryo/KryoOutputFormat.java | 43 --
.../structure/io/kryo/KryoRecordReader.java | 113 ------
.../structure/io/kryo/KryoRecordWriter.java | 54 ---
.../structure/io/kryo/VertexStreamIterator.java | 147 -------
.../gremlin/hadoop/HadoopGraphProvider.java | 30 +-
.../io/graphson/VertexStreamIteratorTest.java | 6 +-
.../io/gryo/GryoRecordReaderWriterTest.java | 113 ++++++
.../io/kryo/KryoRecordReaderWriterTest.java | 113 ------
pom.xml | 2 +-
tinkergraph-gremlin/pom.xml | 6 +-
.../tinkergraph/structure/TinkerGraphTest.java | 55 ++-
122 files changed, 3266 insertions(+), 3244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/96ffd77c/hadoop-gremlin/conf/giraph-gryo.properties
----------------------------------------------------------------------
diff --cc hadoop-gremlin/conf/giraph-gryo.properties
index 0000000,0000000..7813ad7
new file mode 100644
--- /dev/null
+++ b/hadoop-gremlin/conf/giraph-gryo.properties
@@@ -1,0 -1,0 +1,31 @@@
++# Licensed to the Apache Software Foundation (ASF) under one
++# or more contributor license agreements. See the NOTICE file
++# distributed with this work for additional information
++# regarding copyright ownership. The ASF licenses this file
++# to you under the Apache License, Version 2.0 (the
++# "License"); you may not use this file except in compliance
++# with the License. You may obtain a copy of the License at
++#
++# http://www.apache.org/licenses/LICENSE-2.0
++#
++# Unless required by applicable law or agreed to in writing,
++# software distributed under the License is distributed on an
++# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++# KIND, either express or implied. See the License for the
++# specific language governing permissions and limitations
++# under the License.
++gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
++gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
++gremlin.hadoop.graphOutputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
++gremlin.hadoop.memoryOutputFormat=org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat
++gremlin.hadoop.deriveMemory=false
++gremlin.hadoop.jarsInDistributedCache=true
++
++gremlin.hadoop.inputLocation=tinkerpop-modern-vertices.kryo
++gremlin.hadoop.outputLocation=output
++#gremlin.vertexProgram=org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram
++#gremlin.traversalVertexProgram.traversalSupplier.type=CLASS
++#gremlin.traversalVertexProgram.traversalSupplier.object=org.apache.tinkerpop.gremlin.hadoop.process.computer.example.TraversalSupplier1
++
++giraph.minWorkers=2
++giraph.maxWorkers=2
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/96ffd77c/hadoop-gremlin/conf/spark-kryo.properties
----------------------------------------------------------------------
diff --cc hadoop-gremlin/conf/spark-kryo.properties
index 483349f,0000000..85426a0
mode 100644,000000..100644
--- a/hadoop-gremlin/conf/spark-kryo.properties
+++ b/hadoop-gremlin/conf/spark-kryo.properties
@@@ -1,38 -1,0 +1,38 @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
- gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.kryo.KryoInputFormat
- gremlin.hadoop.graphOutputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.kryo.KryoOutputFormat
++gremlin.hadoop.graphInputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
++gremlin.hadoop.graphOutputFormat=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat
+gremlin.hadoop.memoryOutputFormat=org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat
+gremlin.hadoop.deriveMemory=false
+gremlin.hadoop.jarsInDistributedCache=false
+
- gremlin.hadoop.inputLocation=hdfs://localhost:9000/user/marko/tinkerpop-modern-vertices.gio
++gremlin.hadoop.inputLocation=hdfs://localhost:9000/user/marko/tinkerpop-modern-vertices.kryo
+gremlin.hadoop.outputLocation=output
+
+# the vertex program to execute
+gremlin.vertexProgram=org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram
+
+# It is possible to provide Spark configuration parameters for use with SparkGraphComputer
+##########################################################################################
+spark.master=local[4]
+spark.executor.memory=1024m
+spark.eventLog.enabled=true
+spark.serializer=org.apache.spark.serializer.JavaSerializer
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/96ffd77c/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertex.java
----------------------------------------------------------------------
diff --cc hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertex.java
index 38f8a61,0000000..0a02156
mode 100644,000000..100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertex.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkVertex.java
@@@ -1,149 -1,0 +1,149 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
+
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
- import org.apache.tinkerpop.gremlin.structure.io.kryo.KryoReader;
- import org.apache.tinkerpop.gremlin.structure.io.kryo.KryoWriter;
++import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
++import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkVertex implements Vertex, Vertex.Iterators, Serializable {
+
- private static KryoWriter KRYO_WRITER = KryoWriter.build().create();
- private static KryoReader KRYO_READER = KryoReader.build().create();
++ private static GryoWriter GRYO_WRITER = GryoWriter.build().create();
++ private static GryoReader GRYO_READER = GryoReader.build().create();
+ private static final String VERTEX_ID = Graph.Hidden.hide("giraph.gremlin.vertexId");
+
+ private transient TinkerVertex vertex;
+ private byte[] vertexBytes;
+
+ public SparkVertex(final TinkerVertex vertex) {
+ this.vertex = vertex;
+ this.vertex.graph().variables().set(VERTEX_ID, this.vertex.id());
+ }
+
+ @Override
+ public Edge addEdge(final String label, final Vertex inVertex, final Object... keyValues) {
+ return this.vertex.addEdge(label, inVertex, keyValues);
+ }
+
+ @Override
+ public Object id() {
+ return this.vertex.id();
+ }
+
+ @Override
+ public String label() {
+ return this.vertex.label();
+ }
+
+ @Override
+ public Graph graph() {
+ return this.vertex.graph();
+ }
+
+ @Override
+ public <V> VertexProperty<V> property(final String key, final V value) {
+ return this.vertex.property(key, value);
+ }
+
+ @Override
+ public void remove() {
+ this.vertex.remove();
+ }
+
+ @Override
+ public Iterators iterators() {
+ return this;
+ }
+
+ @Override
+ public Iterator<Edge> edgeIterator(final Direction direction, final String... edgeLabels) {
+ return this.vertex.iterators().edgeIterator(direction, edgeLabels);
+ }
+
+ @Override
+ public Iterator<Vertex> vertexIterator(final Direction direction, final String... edgeLabels) {
+ return this.vertex.iterators().vertexIterator(direction, edgeLabels);
+ }
+
+ @Override
+ public <V> Iterator<VertexProperty<V>> propertyIterator(final String... propertyKeys) {
+ return this.vertex.iterators().propertyIterator(propertyKeys);
+ }
+
+ ///////////////////////////////
+
+ private void writeObject(final ObjectOutputStream outputStream) throws IOException {
+ this.deflateVertex();
+ outputStream.defaultWriteObject();
+ }
+
+ private void readObject(final ObjectInputStream inputStream) throws IOException, ClassNotFoundException {
+ inputStream.defaultReadObject();
+ this.inflateVertex();
+ }
+
+ private final void inflateVertex() {
+ if (null != this.vertex)
+ return;
+
+ try {
+ final ByteArrayInputStream bis = new ByteArrayInputStream(this.vertexBytes);
+ final TinkerGraph tinkerGraph = TinkerGraph.open();
- KRYO_READER.readGraph(bis, tinkerGraph);
++ GRYO_READER.readGraph(bis, tinkerGraph);
+ bis.close();
+ this.vertexBytes = null;
+ this.vertex = (TinkerVertex) tinkerGraph.iterators().vertexIterator(tinkerGraph.variables().get(VERTEX_ID).get()).next();
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ private final void deflateVertex() {
+ if (null != this.vertexBytes)
+ return;
+
+ try {
+ final ByteArrayOutputStream bos = new ByteArrayOutputStream();
- KRYO_WRITER.writeGraph(bos, this.vertex.graph());
++ GRYO_WRITER.writeGraph(bos, this.vertex.graph());
+ bos.flush();
+ bos.close();
+ this.vertex = null;
+ this.vertexBytes = bos.toByteArray();
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+}