You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/06/07 13:14:44 UTC
[01/18] incubator-tinkerpop git commit: Introduce Kryo shim to
support serializer reuse
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 406956db5 -> 2c7b1f92b
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
new file mode 100644
index 0000000..d5ba90d
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.util.star;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
+
+public class StarGraphSerializer implements SerializerShim<StarGraph> {
+
+ private final Direction edgeDirectionToSerialize;
+ private GraphFilter graphFilter;
+
+ private final static byte VERSION_1 = Byte.MIN_VALUE;
+
+ public StarGraphSerializer(final Direction edgeDirectionToSerialize, final GraphFilter graphFilter) {
+ this.edgeDirectionToSerialize = edgeDirectionToSerialize;
+ this.graphFilter = graphFilter;
+ }
+
+ @Override
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final StarGraph starGraph) {
+ output.writeByte(VERSION_1);
+ kryo.writeObjectOrNull(output, starGraph.edgeProperties, HashMap.class);
+ kryo.writeObjectOrNull(output, starGraph.metaProperties, HashMap.class);
+ kryo.writeClassAndObject(output, starGraph.starVertex.id);
+ kryo.writeObject(output, starGraph.starVertex.label);
+ writeEdges(kryo, output, starGraph, Direction.IN);
+ writeEdges(kryo, output, starGraph, Direction.OUT);
+ kryo.writeObject(output, null != starGraph.starVertex.vertexProperties);
+ if (null != starGraph.starVertex.vertexProperties) {
+ kryo.writeObject(output, starGraph.starVertex.vertexProperties.size());
+ for (final Map.Entry<String, List<VertexProperty>> vertexProperties : starGraph.starVertex.vertexProperties.entrySet()) {
+ kryo.writeObject(output, vertexProperties.getKey());
+ kryo.writeObject(output, vertexProperties.getValue().size());
+ for (final VertexProperty vertexProperty : vertexProperties.getValue()) {
+ kryo.writeClassAndObject(output, vertexProperty.id());
+ kryo.writeClassAndObject(output, vertexProperty.value());
+ }
+ }
+ }
+ }
+
+ /**
+ * If the returned {@link StarGraph} is null, that means that the {@link GraphFilter} filtered the vertex.
+ */
+ @Override
+ public <I extends InputShim> StarGraph read(KryoShim<I, ?> kryo, I input, Class<StarGraph> clazz) {
+ final StarGraph starGraph = StarGraph.open();
+ input.readByte(); // version field ignored for now - for future use with backward compatibility
+ starGraph.edgeProperties = kryo.readObjectOrNull(input, HashMap.class);
+ starGraph.metaProperties = kryo.readObjectOrNull(input, HashMap.class);
+ starGraph.addVertex(T.id, kryo.readClassAndObject(input), T.label, kryo.readObject(input, String.class));
+ readEdges(kryo, input, starGraph, Direction.IN);
+ readEdges(kryo, input, starGraph, Direction.OUT);
+ if (kryo.readObject(input, Boolean.class)) {
+ final int numberOfUniqueKeys = kryo.readObject(input, Integer.class);
+ for (int i = 0; i < numberOfUniqueKeys; i++) {
+ final String vertexPropertyKey = kryo.readObject(input, String.class);
+ final int numberOfVertexPropertiesWithKey = kryo.readObject(input, Integer.class);
+ for (int j = 0; j < numberOfVertexPropertiesWithKey; j++) {
+ final Object id = kryo.readClassAndObject(input);
+ final Object value = kryo.readClassAndObject(input);
+ starGraph.starVertex.property(VertexProperty.Cardinality.list, vertexPropertyKey, value, T.id, id);
+ }
+ }
+ }
+ return this.graphFilter.hasFilter() ? starGraph.applyGraphFilter(this.graphFilter).orElse(null) : starGraph;
+ }
+
+ private <O extends OutputShim> void writeEdges(final KryoShim<?, O> kryo, final O output, final StarGraph starGraph, final Direction direction) {
+ // only write edges if there are some AND if the user requested them to be serialized AND if they match
+ // the direction being serialized by the format
+ final Map<String, List<Edge>> starEdges = direction.equals(Direction.OUT) ? starGraph.starVertex.outEdges : starGraph.starVertex.inEdges;
+ final boolean writeEdges = null != starEdges && edgeDirectionToSerialize != null
+ && (edgeDirectionToSerialize == direction || edgeDirectionToSerialize == Direction.BOTH);
+ kryo.writeObject(output, writeEdges);
+ if (writeEdges) {
+ kryo.writeObject(output, starEdges.size());
+ for (final Map.Entry<String, List<Edge>> edges : starEdges.entrySet()) {
+ kryo.writeObject(output, edges.getKey());
+ kryo.writeObject(output, edges.getValue().size());
+ for (final Edge edge : edges.getValue()) {
+ kryo.writeClassAndObject(output, edge.id());
+ kryo.writeClassAndObject(output, direction.equals(Direction.OUT) ? edge.inVertex().id() : edge.outVertex().id());
+ }
+ }
+ }
+ }
+
+ private <I extends InputShim> void readEdges(final KryoShim<I, ?> kryo, final I input, final StarGraph starGraph, final Direction direction) {
+ if (kryo.readObject(input, Boolean.class)) {
+ final int numberOfUniqueLabels = kryo.readObject(input, Integer.class);
+ for (int i = 0; i < numberOfUniqueLabels; i++) {
+ final String edgeLabel = kryo.readObject(input, String.class);
+ final int numberOfEdgesWithLabel = kryo.readObject(input, Integer.class);
+ for (int j = 0; j < numberOfEdgesWithLabel; j++) {
+ final Object edgeId = kryo.readClassAndObject(input);
+ final Object adjacentVertexId = kryo.readClassAndObject(input);
+ if (this.graphFilter.checkEdgeLegality(direction, edgeLabel).positive()) {
+ if (direction.equals(Direction.OUT))
+ starGraph.starVertex.addOutEdge(edgeLabel, starGraph.addVertex(T.id, adjacentVertexId), T.id, edgeId);
+ else
+ starGraph.starVertex.addInEdge(edgeLabel, starGraph.addVertex(T.id, adjacentVertexId), T.id, edgeId);
+ } else if (null != starGraph.edgeProperties) {
+ starGraph.edgeProperties.remove(edgeId);
+ }
+ }
+ }
+ }
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
new file mode 100644
index 0000000..2053280
--- /dev/null
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class HadoopPoolShimService implements KryoShimService {
+
+ public Object readClassAndObject(InputStream source) {
+
+ Kryo k = null;
+
+ try {
+ k = HadoopPools.getGryoPool().takeKryo();
+
+ return k.readClassAndObject(new Input(source));
+ } finally {
+ if (null != k) {
+ HadoopPools.getGryoPool().offerKryo(k);
+ }
+ }
+ }
+
+ public void writeClassAndObject(Object o, OutputStream sink) {
+
+ Kryo k = null;
+
+ try {
+ k = HadoopPools.getGryoPool().takeKryo();
+
+ Output output = new Output(sink);
+
+ k.writeClassAndObject(output, o);
+
+ output.flush();
+ } finally {
+ if (null != k) {
+ HadoopPools.getGryoPool().offerKryo(k);
+ }
+ }
+ }
+
+ @Override
+ public int getPriority() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
index f09f703..5074ad5 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPools.java
@@ -43,6 +43,7 @@ public final class HadoopPools {
GRYO_POOL = GryoPool.build().
poolSize(configuration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, 256)).
ioRegistries(configuration.getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).
+ initializeMapper(m -> m.registrationRequired(false)).
create();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
index 9a07f75..e7a38a5 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
@@ -21,6 +21,8 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimServiceLoader;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -63,29 +65,14 @@ public final class ObjectWritable<T> implements WritableComparable<ObjectWritabl
@Override
public void readFields(final DataInput input) throws IOException {
- this.t = HadoopPools.getGryoPool().doWithReader(gryoReader -> {
- try {
- // class argument is Object because gryo doesn't really care that we don't know the specific type.
- // the type is embedded in the stream so it can just read it from there and return it as needed.
- // presumably that will cast nicely to T
- return (T) gryoReader.readObject(new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input)), Object.class);
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- });
+ ByteArrayInputStream bais = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
+ this.t = KryoShimServiceLoader.readClassAndObject(bais);
}
@Override
public void write(final DataOutput output) throws IOException {
- HadoopPools.getGryoPool().doWithWriter(gryoWriter -> {
- try {
- final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- gryoWriter.writeObject(outputStream, this.t);
- WritableUtils.writeCompressedByteArray(output, outputStream.toByteArray());
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- });
+ byte serialized[] = KryoShimServiceLoader.writeClassAndObjectToBytes(this.t);
+ WritableUtils.writeCompressedByteArray(output, serialized);
}
private void writeObject(final ObjectOutputStream outputStream) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
index ac360e9..7ac8e8c 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
@@ -60,42 +61,15 @@ public final class VertexWritable implements Writable, Serializable {
@Override
public void readFields(final DataInput input) throws IOException {
- try {
- this.vertex = null;
- this.vertex = HadoopPools.getGryoPool().doWithReader(gryoReader -> {
- try {
- final ByteArrayInputStream inputStream = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
- return gryoReader.readObject(inputStream, StarGraph.class).getStarVertex(); // read the star graph
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- });
- } catch (final IllegalStateException e) {
- if (e.getCause() instanceof IOException)
- throw (IOException) e.getCause();
- else
- throw e;
- }
+ this.vertex = null;
+ ByteArrayInputStream bais = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
+ this.vertex = ((StarGraph)KryoShimServiceLoader.readClassAndObject(bais)).getStarVertex(); // read the star graph;
}
@Override
public void write(final DataOutput output) throws IOException {
- try {
- HadoopPools.getGryoPool().doWithWriter(gryoWriter -> {
- try {
- final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- gryoWriter.writeObject(outputStream, this.vertex.graph()); // write the star graph
- WritableUtils.writeCompressedByteArray(output, outputStream.toByteArray());
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- });
- } catch (final IllegalStateException e) {
- if (e.getCause() instanceof IOException)
- throw (IOException) e.getCause();
- else
- throw e;
- }
+ byte serialized[] = KryoShimServiceLoader.writeClassAndObjectToBytes(this.vertex.graph());
+ WritableUtils.writeCompressedByteArray(output, serialized);
}
private void writeObject(final ObjectOutputStream outputStream) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService b/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService
new file mode 100644
index 0000000..0b27e72
--- /dev/null
+++ b/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService
@@ -0,0 +1 @@
+org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService # HadoopPools provides/caches instances of TinkerPop's shaded Kryo
[14/18] incubator-tinkerpop git commit: made SparkHadoopGraphProvider
more robust.
Posted by ok...@apache.org.
made SparkHadoopGraphProvider more robust.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/08040f80
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/08040f80
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/08040f80
Branch: refs/heads/master
Commit: 08040f8083aecb5b18eafbdf367f93b72a5356f0
Parents: 359f04b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Jun 6 18:47:45 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Jun 6 18:47:45 2016 -0600
----------------------------------------------------------------------
.../spark/process/computer/SparkHadoopGraphProvider.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08040f80/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 7737d1e..b1f610d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -33,8 +33,10 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyPageRankTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyPeerPressureTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyProgramTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PeerPressureTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest;
import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorageCheck;
@@ -58,10 +60,13 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
+ config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT,
+ !test.equals(ProgramTest.Traversals.class) && !test.equals(GroovyProgramTest.class)); // this makes the test suite go really fast
// toy graph inputRDD does not have corresponding outputRDD so where jobs chain, it fails (failing makes sense)
if (null != loadGraphWith &&
+ !test.equals(ProgramTest.Traversals.class) &&
+ !test.equals(GroovyProgramTest.Traversals.class) &&
!test.equals(PageRankTest.Traversals.class) &&
!test.equals(GroovyPageRankTest.Traversals.class) &&
!test.equals(PeerPressureTest.Traversals.class) &&
[03/18] incubator-tinkerpop git commit: Kryo shim refactoring and
documentation
Posted by ok...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
index d5ba90d..431e1eb 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
@@ -27,11 +27,21 @@ import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+/**
+ * Kryo serializer for {@link StarGraph}. Implements an internal versioning capability for backward compatibility.
+ * The single byte at the front of the serialization stream denotes the version. That version can be used to choose
+ * the correct deserialization mechanism. The limitation is that this versioning won't help with backward
+ * compatibility for custom serializers from providers. Providers should be encouraged to write their serializers
+ * with backward compatibility in mind.
+ *
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
public class StarGraphSerializer implements SerializerShim<StarGraph> {
private final Direction edgeDirectionToSerialize;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
index 2053280..c19b914 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
@@ -18,7 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
index e7a38a5..88f7ee1 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
@@ -21,11 +21,9 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimServiceLoader;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
index 7ac8e8c..2252ded 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
@@ -21,12 +21,11 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimServiceLoader;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.structure.util.ElementHelper;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService b/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService
new file mode 100644
index 0000000..0b27e72
--- /dev/null
+++ b/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService
@@ -0,0 +1 @@
+org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService # HadoopPools provides/caches instances of TinkerPop's shaded Kryo
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService b/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService
deleted file mode 100644
index 0b27e72..0000000
--- a/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShimService
+++ /dev/null
@@ -1 +0,0 @@
-org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService # HadoopPools provides/caches instances of TinkerPop's shaded Kryo
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java
new file mode 100644
index 0000000..4c99e70
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.serializers.JavaSerializer;
+import com.google.common.base.Preconditions;
+import org.apache.spark.serializer.KryoRegistrator;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderGlobalStep;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalExplanation;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.ObjectWritableSerializer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.VertexWritableSerializer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedSerializerAdapter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.TypeRegistration;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A spark.kryo.registrator implementation that installs TinkerPop types.
+ * This is intended for use with spark.serializer=KryoSerializer, not GryoSerializer.
+ */
+public class TinkerPopKryoRegistrator implements KryoRegistrator {
+
+ private static final Logger log = LoggerFactory.getLogger(TinkerPopKryoRegistrator.class);
+
+ @Override
+ public void registerClasses(Kryo kryo) {
+ // TinkerPop type registrations copied from GyroSerializer's constructor
+ kryo.register(MessagePayload.class);
+ kryo.register(ViewIncomingPayload.class);
+ kryo.register(ViewOutgoingPayload.class);
+ kryo.register(ViewPayload.class);
+ kryo.register(VertexWritable.class, new UnshadedSerializerAdapter<>(new VertexWritableSerializer()));
+ kryo.register(ObjectWritable.class, new UnshadedSerializerAdapter<>(new ObjectWritableSerializer<>()));
+
+ Set<Class<?>> shimmedClasses = new HashSet<>();
+
+ Set<Class<?>> javaSerializationClasses = new HashSet<>();
+
+ // Copy GryoMapper's default registrations
+ for (TypeRegistration<?> tr : GryoMapper.build().create().getTypeRegistrations()) {
+ // Special case for JavaSerializer, which is generally implemented in terms of TinkerPop's
+ // problematic static GryoMapper/GryoSerializer pool (these are handled below the loop)
+ org.apache.tinkerpop.shaded.kryo.Serializer<?> shadedSerializer = tr.getShadedSerializer();
+ SerializerShim<?> serializerShim = tr.getSerializerShim();
+ if (null != shadedSerializer &&
+ shadedSerializer.getClass().equals(org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer.class)) {
+ javaSerializationClasses.add(tr.getTargetClass());
+ } else if (null != serializerShim) {
+ log.debug("Registering class {} to serializer shim {} (serializer shim class {})",
+ tr.getTargetClass(), serializerShim, serializerShim.getClass());
+ kryo.register(tr.getTargetClass(), new UnshadedSerializerAdapter<>(serializerShim));
+ shimmedClasses.add(tr.getTargetClass());
+ } else {
+ // Register with the default behavior (FieldSerializer)
+ log.debug("Registering class {} with default serializer", tr.getTargetClass());
+ kryo.register(tr.getTargetClass());
+ }
+ }
+
+ Map<Class<?>, Serializer<?>> javaSerializerReplacements = new HashMap<>();
+ javaSerializerReplacements.put(GroupStep.GroupBiOperator.class, new JavaSerializer());
+ javaSerializerReplacements.put(OrderGlobalStep.OrderBiOperator.class, null);
+ javaSerializerReplacements.put(TraversalExplanation.class, null);
+
+ for (Map.Entry<Class<?>, Serializer<?>> e : javaSerializerReplacements.entrySet()) {
+ Class<?> c = e.getKey();
+ Serializer<?> s = e.getValue();
+
+ if (javaSerializationClasses.remove(c)) {
+ if (null != s) {
+ log.debug("Registering class {} with serializer {}", c, s);
+ kryo.register(c, s);
+ } else {
+ log.debug("Registering class {} with default serializer", c);
+ kryo.register(c);
+ }
+ } else {
+ log.debug("Registering class {} with JavaSerializer", c);
+ kryo.register(c, new JavaSerializer());
+ }
+ }
+
+ // We really care about StarGraph's shim serializer, so make sure we registered it
+ if (!shimmedClasses.contains(StarGraph.class)) {
+ log.warn("No SerializerShim found for StarGraph");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
index 21cbc60..4ceb045 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
@@ -20,6 +20,10 @@
package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Serializer;
import org.apache.tinkerpop.shaded.kryo.io.Input;
@@ -28,16 +32,16 @@ import org.apache.tinkerpop.shaded.kryo.io.Output;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class ObjectWritableSerializer<T> extends Serializer<ObjectWritable<T>> {
+public final class ObjectWritableSerializer<T> implements SerializerShim<ObjectWritable<T>> {
@Override
- public void write(final Kryo kryo, final Output output, final ObjectWritable<T> objectWritable) {
- kryo.writeClassAndObject(output, objectWritable.get());
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, ObjectWritable<T> starGraph) {
+ kryo.writeClassAndObject(output, starGraph.get());
output.flush();
}
@Override
- public ObjectWritable<T> read(final Kryo kryo, final Input input, final Class<ObjectWritable<T>> clazz) {
+ public <I extends InputShim> ObjectWritable<T> read(KryoShim<I, ?> kryo, I input, Class<ObjectWritable<T>> clazz) {
return new ObjectWritable(kryo.readClassAndObject(input));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
index 97891f3..f3c1b15 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
@@ -20,6 +20,10 @@
package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Serializer;
@@ -29,14 +33,16 @@ import org.apache.tinkerpop.shaded.kryo.io.Output;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class VertexWritableSerializer extends Serializer<VertexWritable> {
+public final class VertexWritableSerializer implements SerializerShim<VertexWritable> {
+
@Override
- public void write(final Kryo kryo, final Output output, final VertexWritable vertexWritable) {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, VertexWritable vertexWritable) {
kryo.writeObject(output, vertexWritable.get().graph());
+ output.flush();
}
@Override
- public VertexWritable read(final Kryo kryo, final Input input, final Class<VertexWritable> aClass) {
+ public <I extends InputShim> VertexWritable read(KryoShim<I, ?> kryo, I input, Class<VertexWritable> clazz) {
return new VertexWritable(kryo.readObject(input, StarGraph.class).getStarVertex());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java
new file mode 100644
index 0000000..c533af7
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded;
+
+import com.esotericsoftware.kryo.io.Input;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+
+public class UnshadedInputAdapter implements InputShim
+{
+
+ private final Input unshadedInput;
+
+ public UnshadedInputAdapter(Input unshadedInput)
+ {
+ this.unshadedInput = unshadedInput;
+ }
+
+ Input getUnshadedInput()
+ {
+ return unshadedInput;
+ }
+
+ @Override
+ public byte readByte()
+ {
+ return unshadedInput.readByte();
+ }
+
+ @Override
+ public byte[] readBytes(int size) {
+ return unshadedInput.readBytes(size);
+ }
+
+ @Override
+ public String readString()
+ {
+ return unshadedInput.readString();
+ }
+
+ @Override
+ public long readLong()
+ {
+ return unshadedInput.readLong();
+ }
+
+ @Override
+ public int readInt() {
+ return unshadedInput.readInt();
+ }
+
+ @Override
+ public double readDouble()
+ {
+ return unshadedInput.readDouble();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java
new file mode 100644
index 0000000..b14abe0
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded;
+
+import com.esotericsoftware.kryo.Kryo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+
+public class UnshadedKryoAdapter implements KryoShim<UnshadedInputAdapter, UnshadedOutputAdapter>
+{
+ private final Kryo unshadedKryo;
+
+ public UnshadedKryoAdapter(Kryo unshadedKryo)
+ {
+ this.unshadedKryo = unshadedKryo;
+ }
+
+ @Override
+ public <T> T readObject(UnshadedInputAdapter input, Class<T> type)
+ {
+ return unshadedKryo.readObject(input.getUnshadedInput(), type);
+ }
+
+ @Override
+ public Object readClassAndObject(UnshadedInputAdapter input)
+ {
+ return unshadedKryo.readClassAndObject(input.getUnshadedInput());
+ }
+
+ @Override
+ public void writeObject(UnshadedOutputAdapter output, Object object)
+ {
+ unshadedKryo.writeObject(output.getUnshadedOutput(), object);
+ }
+
+ @Override
+ public void writeClassAndObject(UnshadedOutputAdapter output, Object object)
+ {
+ unshadedKryo.writeClassAndObject(output.getUnshadedOutput(), object);
+ }
+
+ @Override
+ public <T> T readObjectOrNull(UnshadedInputAdapter input, Class<T> type)
+ {
+ return unshadedKryo.readObjectOrNull(input.getUnshadedInput(), type);
+ }
+
+ @Override
+ public void writeObjectOrNull(UnshadedOutputAdapter output, Object object, Class type)
+ {
+ unshadedKryo.writeObjectOrNull(output.getUnshadedOutput(), object, type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
new file mode 100644
index 0000000..d0411e8
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Copyright DataStax, Inc.
+ * <p>
+ * Please see the included license file for details.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded;
+
+import com.twitter.chill.KryoInstantiator;
+import com.twitter.chill.KryoPool;
+import com.twitter.chill.SerDeState;
+import org.apache.spark.SparkConf;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.TinkerPopKryoRegistrator;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class UnshadedKryoShimService implements KryoShimService {
+
+ public static final String SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY = "tinkerpop.kryo.poolsize";
+
+ private static final Logger log = LoggerFactory.getLogger(UnshadedKryoShimService.class);
+ private static final int SPARK_KRYO_POOL_SIZE_DEFAULT = 8;
+
+ private final KryoSerializer sparkKryoSerializer;
+ private final KryoPool kryoPool;
+
+ public UnshadedKryoShimService() {
+ this(TinkerPopKryoRegistrator.class.getCanonicalName(), getDefaultKryoPoolSize());
+ }
+
+ public UnshadedKryoShimService(String sparkKryoRegistratorClassname, int kryoPoolSize) {
+ SparkConf sparkConf = new SparkConf();
+ sparkConf.set("spark.serializer", KryoSerializer.class.getCanonicalName());
+ sparkConf.set("spark.kryo.registrator", sparkKryoRegistratorClassname);
+ sparkKryoSerializer = new KryoSerializer(sparkConf);
+ kryoPool = KryoPool.withByteArrayOutputStream(kryoPoolSize, new KryoInstantiator());
+ }
+
+ @Override
+ public Object readClassAndObject(InputStream source) {
+ SerDeState sds = null;
+ try {
+ sds = kryoPool.borrow();
+
+ sds.setInput(source);
+
+ return sds.readClassAndObject();
+ } finally {
+ kryoPool.release(sds);
+ }
+ }
+
+ @Override
+ public void writeClassAndObject(Object o, OutputStream sink) {
+ SerDeState sds = null;
+ try {
+ sds = kryoPool.borrow();
+
+ sds.writeClassAndObject(o); // this writes to an internal buffer
+
+ sds.writeOutputTo(sink); // this copies the internal buffer to sink
+
+ sink.flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ kryoPool.release(sds);
+ }
+ }
+
+ @Override
+ public int getPriority() {
+ return 1024;
+ }
+
+ private static int getDefaultKryoPoolSize() {
+ String raw = System.getProperty(SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY);
+
+ int size = SPARK_KRYO_POOL_SIZE_DEFAULT;
+ try {
+ size = Integer.valueOf(raw);
+ log.info("Setting kryo pool size to {} according to system property {}", size,
+ SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY);
+ } catch (NumberFormatException e) {
+ log.error("System property {}={} could not be parsed as an integer, using default value {}",
+ SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY, raw, size, e);
+ }
+
+ return size;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java
new file mode 100644
index 0000000..9cc59d4
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded;
+
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+
+public class UnshadedOutputAdapter implements OutputShim
+{
+ private final Output unshadedOutput;
+
+ public UnshadedOutputAdapter(Output unshadedOutput)
+ {
+ this.unshadedOutput = unshadedOutput;
+ }
+
+ Output getUnshadedOutput()
+ {
+ return unshadedOutput;
+ }
+
+ @Override
+ public void writeByte(byte b)
+ {
+ unshadedOutput.writeByte(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] array, int offset, int count) {
+ unshadedOutput.writeBytes(array, offset, count);
+ }
+
+ @Override
+ public void writeString(String s)
+ {
+ unshadedOutput.writeString(s);
+ }
+
+ @Override
+ public void writeLong(long l)
+ {
+ unshadedOutput.writeLong(l);
+ }
+
+ @Override
+ public void writeInt(int i) {
+ unshadedOutput.writeInt(i);
+ }
+
+ @Override
+ public void writeDouble(double d)
+ {
+ unshadedOutput.writeDouble(d);
+ }
+
+ @Override
+ public void flush()
+ {
+ unshadedOutput.flush();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java
new file mode 100644
index 0000000..efc9a4f
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Copyright DataStax, Inc.
+ *
+ * Please see the included license file for details.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+
+public class UnshadedSerializerAdapter<T> extends Serializer<T>
+{
+
+ SerializerShim<T> serializer;
+
+ public UnshadedSerializerAdapter(SerializerShim<T> serializer) {
+ this.serializer = serializer;
+ setImmutable(this.serializer.isImmutable());
+ }
+
+ @Override
+ public void write(Kryo kryo, Output output, T t) {
+ UnshadedKryoAdapter shadedKryoAdapter = new UnshadedKryoAdapter(kryo);
+ UnshadedOutputAdapter shadedOutputAdapter = new UnshadedOutputAdapter(output);
+ serializer.write(shadedKryoAdapter, shadedOutputAdapter, t);
+ }
+
+ @Override
+ public T read(Kryo kryo, Input input, Class<T> aClass)
+ {
+ UnshadedKryoAdapter shadedKryoAdapter = new UnshadedKryoAdapter(kryo);
+ UnshadedInputAdapter shadedInputAdapter = new UnshadedInputAdapter(input);
+ return serializer.read(shadedKryoAdapter, shadedInputAdapter, aClass);
+ }
+}
[06/18] incubator-tinkerpop git commit: Kryo shim configuration tweaks
Posted by ok...@apache.org.
Kryo shim configuration tweaks
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/9321a3e1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/9321a3e1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/9321a3e1
Branch: refs/heads/master
Commit: 9321a3e14eab4ed05f8ef5f4c77de481a4011b81
Parents: 218d790
Author: Dan LaRocque <da...@hopcount.org>
Authored: Mon Jun 6 02:24:12 2016 -0400
Committer: Dan LaRocque <da...@hopcount.org>
Committed: Mon Jun 6 03:10:03 2016 -0400
----------------------------------------------------------------------
.../process/computer/GiraphWorkerContext.java | 3 +-
.../gremlin/structure/io/gryo/GryoMapper.java | 30 ++-
.../gremlin/structure/io/gryo/GryoPool.java | 1 +
.../structure/io/gryo/GryoSerializers.java | 40 ++--
.../structure/io/gryo/JavaTimeSerializers.java | 127 +++++-------
.../structure/io/gryo/PairSerializer.java | 11 +-
.../structure/io/gryo/TypeRegistration.java | 12 ++
.../io/gryo/kryoshim/KryoShimService.java | 16 ++
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 23 ++-
.../io/gryo/kryoshim/SerializerShim.java | 2 +-
.../hadoop/process/computer/HadoopCombine.java | 3 +-
.../hadoop/process/computer/HadoopMap.java | 3 +-
.../hadoop/process/computer/HadoopReduce.java | 3 +-
.../structure/io/HadoopPoolShimService.java | 7 +
.../structure/io/HadoopPoolsConfigurable.java | 4 +-
.../structure/io/gryo/GryoRecordReader.java | 3 +-
.../structure/io/gryo/GryoRecordWriter.java | 4 +-
.../spark/process/computer/SparkExecutor.java | 3 +-
.../structure/io/TinkerPopKryoRegistrator.java | 121 ------------
.../spark/structure/io/gryo/GryoSerializer.java | 2 +-
.../io/gryo/IoRegistryAwareKryoSerializer.java | 116 +++++++++++
.../io/gryo/TinkerPopKryoRegistrator.java | 194 +++++++++++++++++++
.../unshaded/UnshadedKryoShimService.java | 131 ++++++++-----
...n.structure.io.gryo.kryoshim.KryoShimService | 1 +
.../spark/structure/io/ToyGraphInputRDD.java | 3 +-
25 files changed, 572 insertions(+), 291 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
index 86b733c..0122ab4 100644
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
@@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.ImmutableMemory;
import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.util.Iterator;
@@ -45,7 +46,7 @@ public final class GiraphWorkerContext extends WorkerContext {
public void preApplication() throws InstantiationException, IllegalAccessException {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration());
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
final VertexProgram vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
this.memory = new GiraphMemory(this, vertexProgram);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 851b03c..41ca44d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
+import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
@@ -369,6 +370,20 @@ public final class GryoMapper implements Mapper<Kryo> {
private Supplier<ClassResolver> classResolver = GryoClassResolver::new;
private Builder() {
+ // Validate the default registrations
+ // For justification of these default registration rules, see TinkerPopKryoRegistrator
+ for (TypeRegistration<?> tr : typeRegistrations) {
+ if (tr.hasSerializer() /* no serializer is acceptable */ &&
+ null == tr.getSerializerShim() /* a shim serializer is acceptable */ &&
+ !(tr.getShadedSerializer() instanceof JavaSerializer) /* shaded JavaSerializer is acceptable */) {
+ // everything else is invalid
+ String msg = String.format("The default GryoMapper type registration %s is invalid. " +
+ "It must supply either an implementation of %s or %s, but supplies neither. " +
+ "This is probably a bug in GryoMapper's default serialization class registrations.", tr,
+ SerializerShim.class.getCanonicalName(), JavaSerializer.class.getCanonicalName());
+ throw new IllegalStateException(msg);
+ }
+ }
}
/**
@@ -538,8 +553,8 @@ public final class GryoMapper implements Mapper<Kryo> {
if (1 < serializerCount) {
String msg = String.format(
"GryoTypeReg accepts at most one kind of serializer, but multiple " +
- "serializers were supplied for class %s (id %s). " +
- "Shaded serializer: %s. Shim serializer: %s. Shaded serializer function: %s.",
+ "serializers were supplied for class %s (id %s). " +
+ "Shaded serializer: %s. Shim serializer: %s. Shaded serializer function: %s.",
this.clazz.getCanonicalName(), id,
this.shadedSerializer, this.serializerShim, this.functionOfShadedKryo);
throw new IllegalArgumentException(msg);
@@ -603,5 +618,16 @@ public final class GryoMapper implements Mapper<Kryo> {
return kryo;
}
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this)
+ .append("targetClass", clazz)
+ .append("id", id)
+ .append("shadedSerializer", shadedSerializer)
+ .append("serializerShim", serializerShim)
+ .append("functionOfShadedKryo", functionOfShadedKryo)
+ .toString();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
index e7bf636..59f8a5d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoPool.java
@@ -40,6 +40,7 @@ import java.util.function.Function;
public final class GryoPool {
public static final String CONFIG_IO_REGISTRY = "gremlin.io.registry";
public static final String CONFIG_IO_GRYO_POOL_SIZE = "gremlin.io.gryo.poolSize";
+ public static final int CONFIG_IO_GRYO_POOL_SIZE_DEFAULT = 256;
public enum Type {READER, WRITER, READER_WRITER}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
index ae99ac6..16fbe85 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
@@ -23,16 +23,16 @@ import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPath;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
/**
* Class used to serialize graph-based objects such as vertices, edges, properties, and paths. These objects are
@@ -42,19 +42,19 @@ import org.apache.tinkerpop.shaded.kryo.io.Output;
* @author Stephen Mallette (http://stephen.genoprime.com)
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-final class GryoSerializers {
+public final class GryoSerializers {
/**
* Serializes any {@link Edge} implementation encountered to a {@link DetachedEdge}.
*/
- final static class EdgeSerializer extends Serializer<Edge> {
+ final static class EdgeSerializer implements SerializerShim<Edge> {
@Override
- public void write(final Kryo kryo, final Output output, final Edge edge) {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Edge edge) {
kryo.writeClassAndObject(output, DetachedFactory.detach(edge, true));
}
@Override
- public Edge read(final Kryo kryo, final Input input, final Class<Edge> edgeClass) {
+ public <I extends InputShim> Edge read(KryoShim<I, ?> kryo, I input, Class<Edge> edgeClass) {
final Object o = kryo.readClassAndObject(input);
return (Edge) o;
}
@@ -63,14 +63,14 @@ final class GryoSerializers {
/**
* Serializes any {@link Vertex} implementation encountered to an {@link DetachedVertex}.
*/
- final static class VertexSerializer extends Serializer<Vertex> {
+ final static class VertexSerializer implements SerializerShim<Vertex> {
@Override
- public void write(final Kryo kryo, final Output output, final Vertex vertex) {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Vertex vertex) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertex, true));
}
@Override
- public Vertex read(final Kryo kryo, final Input input, final Class<Vertex> vertexClass) {
+ public <I extends InputShim> Vertex read(KryoShim<I, ?> kryo, I input, Class<Vertex> vertexClass) {
return (Vertex) kryo.readClassAndObject(input);
}
}
@@ -78,14 +78,14 @@ final class GryoSerializers {
/**
* Serializes any {@link Property} implementation encountered to an {@link DetachedProperty}.
*/
- final static class PropertySerializer extends Serializer<Property> {
+ final static class PropertySerializer implements SerializerShim<Property> {
@Override
- public void write(final Kryo kryo, final Output output, final Property property) {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Property property) {
kryo.writeClassAndObject(output, property instanceof VertexProperty ? DetachedFactory.detach((VertexProperty) property, true) : DetachedFactory.detach(property));
}
@Override
- public Property read(final Kryo kryo, final Input input, final Class<Property> propertyClass) {
+ public <I extends InputShim> Property read(KryoShim<I, ?> kryo, I input, Class<Property> propertyClass) {
return (Property) kryo.readClassAndObject(input);
}
}
@@ -93,14 +93,14 @@ final class GryoSerializers {
/**
* Serializes any {@link VertexProperty} implementation encountered to an {@link DetachedVertexProperty}.
*/
- final static class VertexPropertySerializer extends Serializer<VertexProperty> {
+ final static class VertexPropertySerializer implements SerializerShim<VertexProperty> {
@Override
- public void write(final Kryo kryo, final Output output, final VertexProperty vertexProperty) {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, VertexProperty vertexProperty) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertexProperty, true));
}
@Override
- public VertexProperty read(final Kryo kryo, final Input input, final Class<VertexProperty> vertexPropertyClass) {
+ public <I extends InputShim> VertexProperty read(KryoShim<I, ?> kryo, I input, Class<VertexProperty> vertexPropertyClass) {
return (VertexProperty) kryo.readClassAndObject(input);
}
}
@@ -108,14 +108,14 @@ final class GryoSerializers {
/**
* Serializes any {@link Path} implementation encountered to an {@link DetachedPath}.
*/
- final static class PathSerializer extends Serializer<Path> {
+ public final static class PathSerializer implements SerializerShim<Path> {
@Override
- public void write(final Kryo kryo, final Output output, final Path path) {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Path path) {
kryo.writeClassAndObject(output, DetachedFactory.detach(path, false));
}
@Override
- public Path read(final Kryo kryo, final Input input, final Class<Path> pathClass) {
+ public <I extends InputShim> Path read(KryoShim<I, ?> kryo, I input, Class<Path> pathClass) {
return (Path) kryo.readClassAndObject(input);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
index 1d4e236..8b14345 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
@@ -18,10 +18,10 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import java.time.Duration;
import java.time.Instant;
@@ -48,17 +48,14 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link Duration} class.
*/
- final static class DurationSerializer extends Serializer<Duration>
- {
+ final static class DurationSerializer implements SerializerShim<Duration> {
@Override
- public void write(final Kryo kryo, final Output output, final Duration duration)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Duration duration) {
output.writeLong(duration.toNanos());
}
@Override
- public Duration read(final Kryo kryo, final Input input, final Class<Duration> durationClass)
- {
+ public <I extends InputShim> Duration read(KryoShim<I, ?> kryo, I input, Class<Duration> durationClass) {
return Duration.ofNanos(input.readLong());
}
}
@@ -66,18 +63,15 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link Instant} class.
*/
- final static class InstantSerializer extends Serializer<Instant>
- {
+ final static class InstantSerializer implements SerializerShim<Instant> {
@Override
- public void write(Kryo kryo, Output output, Instant instant)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Instant instant) {
output.writeLong(instant.getEpochSecond());
output.writeInt(instant.getNano());
}
@Override
- public Instant read(Kryo kryo, Input input, Class<Instant> aClass)
- {
+ public <I extends InputShim> Instant read(KryoShim<I, ?> kryo, I input, Class<Instant> aClass) {
return Instant.ofEpochSecond(input.readLong(), input.readInt());
}
}
@@ -85,17 +79,14 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link LocalDate} class.
*/
- final static class LocalDateSerializer extends Serializer<LocalDate>
- {
+ final static class LocalDateSerializer implements SerializerShim<LocalDate> {
@Override
- public void write(final Kryo kryo, final Output output, final LocalDate localDate)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, LocalDate localDate) {
output.writeLong(localDate.toEpochDay());
}
@Override
- public LocalDate read(final Kryo kryo, final Input input, final Class<LocalDate> clazz)
- {
+ public <I extends InputShim> LocalDate read(KryoShim<I, ?> kryo, I input, Class<LocalDate> clazz) {
return LocalDate.ofEpochDay(input.readLong());
}
}
@@ -103,11 +94,9 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link LocalDateTime} class.
*/
- final static class LocalDateTimeSerializer extends Serializer<LocalDateTime>
- {
+ final static class LocalDateTimeSerializer implements SerializerShim<LocalDateTime> {
@Override
- public void write(final Kryo kryo, final Output output, final LocalDateTime localDateTime)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, LocalDateTime localDateTime) {
output.writeInt(localDateTime.getYear());
output.writeInt(localDateTime.getMonthValue());
output.writeInt(localDateTime.getDayOfMonth());
@@ -118,8 +107,7 @@ final class JavaTimeSerializers {
}
@Override
- public LocalDateTime read(final Kryo kryo, final Input input, final Class<LocalDateTime> clazz)
- {
+ public <I extends InputShim> LocalDateTime read(KryoShim<I, ?> kryo, I input, Class<LocalDateTime> clazz) {
return LocalDateTime.of(input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt());
}
}
@@ -127,17 +115,14 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link LocalTime} class.
*/
- final static class LocalTimeSerializer extends Serializer<LocalTime>
- {
+ final static class LocalTimeSerializer implements SerializerShim<LocalTime> {
@Override
- public void write(final Kryo kryo, final Output output, final LocalTime localTime)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, LocalTime localTime) {
output.writeLong(localTime.toNanoOfDay());
}
@Override
- public LocalTime read(final Kryo kryo, final Input input, final Class<LocalTime> clazz)
- {
+ public <I extends InputShim> LocalTime read(KryoShim<I, ?> kryo, I input, Class<LocalTime> clazz) {
return LocalTime.ofNanoOfDay(input.readLong());
}
}
@@ -145,37 +130,31 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link MonthDay} class.
*/
- final static class MonthDaySerializer extends Serializer<MonthDay>
- {
+ final static class MonthDaySerializer implements SerializerShim<MonthDay> {
@Override
- public void write(final Kryo kryo, final Output output, final MonthDay monthDay)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, MonthDay monthDay) {
output.writeInt(monthDay.getMonthValue());
output.writeInt(monthDay.getDayOfMonth());
}
@Override
- public MonthDay read(final Kryo kryo, final Input input, final Class<MonthDay> clazz)
- {
- return MonthDay.of(input.readInt(), input.readInt());
+ public <I extends InputShim> MonthDay read(KryoShim<I, ?> kryo, I input, Class<MonthDay> clazz) {
+ return null;
}
}
/**
* Serializer for the {@link OffsetDateTime} class.
*/
- final static class OffsetDateTimeSerializer extends Serializer<OffsetDateTime>
- {
+ final static class OffsetDateTimeSerializer implements SerializerShim<OffsetDateTime> {
@Override
- public void write(final Kryo kryo, final Output output, final OffsetDateTime offsetDateTime)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, OffsetDateTime offsetDateTime) {
kryo.writeObject(output, offsetDateTime.toLocalDateTime());
kryo.writeObject(output, offsetDateTime.getOffset());
}
@Override
- public OffsetDateTime read(final Kryo kryo, final Input input, final Class<OffsetDateTime> clazz)
- {
+ public <I extends InputShim> OffsetDateTime read(KryoShim<I, ?> kryo, I input, Class<OffsetDateTime> clazz) {
return OffsetDateTime.of(kryo.readObject(input, LocalDateTime.class), kryo.readObject(input, ZoneOffset.class));
}
}
@@ -183,18 +162,15 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link OffsetTime} class.
*/
- final static class OffsetTimeSerializer extends Serializer<OffsetTime>
- {
+ final static class OffsetTimeSerializer implements SerializerShim<OffsetTime> {
@Override
- public void write(final Kryo kryo, final Output output, final OffsetTime offsetTime)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, OffsetTime offsetTime) {
kryo.writeObject(output, offsetTime.toLocalTime());
kryo.writeObject(output, offsetTime.getOffset());
}
@Override
- public OffsetTime read(final Kryo kryo, final Input input, final Class<OffsetTime> clazz)
- {
+ public <I extends InputShim> OffsetTime read(KryoShim<I, ?> kryo, I input, Class<OffsetTime> clazz) {
return OffsetTime.of(kryo.readObject(input, LocalTime.class), kryo.readObject(input, ZoneOffset.class));
}
}
@@ -202,19 +178,16 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link Period} class.
*/
- final static class PeriodSerializer extends Serializer<Period>
- {
+ final static class PeriodSerializer implements SerializerShim<Period> {
@Override
- public void write(final Kryo kryo, final Output output, final Period period)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Period period) {
output.writeInt(period.getYears());
output.writeInt(period.getMonths());
output.writeInt(period.getDays());
}
@Override
- public Period read(final Kryo kryo, final Input input, final Class<Period> clazz)
- {
+ public <I extends InputShim> Period read(KryoShim<I, ?> kryo, I input, Class<Period> clazz) {
return Period.of(input.readInt(), input.readInt(), input.readInt());
}
}
@@ -222,17 +195,14 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link Year} class.
*/
- final static class YearSerializer extends Serializer<Year>
- {
+ final static class YearSerializer implements SerializerShim<Year> {
@Override
- public void write(final Kryo kryo, final Output output, final Year year)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Year year) {
output.writeInt(year.getValue());
}
@Override
- public Year read(final Kryo kryo, final Input input, final Class<Year> clazz)
- {
+ public <I extends InputShim> Year read(KryoShim<I, ?> kryo, I input, Class<Year> clazz) {
return Year.of(input.readInt());
}
}
@@ -240,18 +210,15 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link YearMonth} class.
*/
- final static class YearMonthSerializer extends Serializer<YearMonth>
- {
+ final static class YearMonthSerializer implements SerializerShim<YearMonth> {
@Override
- public void write(final Kryo kryo, final Output output, final YearMonth monthDay)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, YearMonth monthDay) {
output.writeInt(monthDay.getYear());
output.writeInt(monthDay.getMonthValue());
}
@Override
- public YearMonth read(final Kryo kryo, final Input input, final Class<YearMonth> clazz)
- {
+ public <I extends InputShim> YearMonth read(KryoShim<I, ?> kryo, I input, Class<YearMonth> clazz) {
return YearMonth.of(input.readInt(), input.readInt());
}
}
@@ -259,11 +226,9 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link ZonedDateTime} class.
*/
- final static class ZonedDateTimeSerializer extends Serializer<ZonedDateTime>
- {
+ final static class ZonedDateTimeSerializer implements SerializerShim<ZonedDateTime> {
@Override
- public void write(final Kryo kryo, final Output output, final ZonedDateTime zonedDateTime)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, ZonedDateTime zonedDateTime) {
output.writeInt(zonedDateTime.getYear());
output.writeInt(zonedDateTime.getMonthValue());
output.writeInt(zonedDateTime.getDayOfMonth());
@@ -275,8 +240,7 @@ final class JavaTimeSerializers {
}
@Override
- public ZonedDateTime read(final Kryo kryo, final Input input, final Class<ZonedDateTime> clazz)
- {
+ public <I extends InputShim> ZonedDateTime read(KryoShim<I, ?> kryo, I input, Class<ZonedDateTime> clazz) {
return ZonedDateTime.of(input.readInt(), input.readInt(), input.readInt(),
input.readInt(), input.readInt(), input.readInt(), input.readInt(),
ZoneId.of(input.readString()));
@@ -286,17 +250,14 @@ final class JavaTimeSerializers {
/**
* Serializer for the {@link ZoneOffset} class.
*/
- final static class ZoneOffsetSerializer extends Serializer<ZoneOffset>
- {
+ final static class ZoneOffsetSerializer implements SerializerShim<ZoneOffset> {
@Override
- public void write(final Kryo kryo, final Output output, final ZoneOffset zoneOffset)
- {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, ZoneOffset zoneOffset) {
output.writeString(zoneOffset.getId());
}
@Override
- public ZoneOffset read(final Kryo kryo, final Input input, final Class<ZoneOffset> clazz)
- {
+ public <I extends InputShim> ZoneOffset read(KryoShim<I, ?> kryo, I input, Class<ZoneOffset> clazz) {
return ZoneOffset.of(input.readString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
index e5e92e7..0464b22 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
@@ -18,6 +18,10 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Serializer;
import org.apache.tinkerpop.shaded.kryo.io.Input;
@@ -27,16 +31,15 @@ import org.javatuples.Pair;
/**
* @author Daniel Kuppitz (http://gremlin.guru)
*/
-final class PairSerializer extends Serializer<Pair> {
-
+final class PairSerializer implements SerializerShim<Pair> {
@Override
- public void write(final Kryo kryo, final Output output, final Pair pair) {
+ public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Pair pair) {
kryo.writeClassAndObject(output, pair.getValue0());
kryo.writeClassAndObject(output, pair.getValue1());
}
@Override
- public Pair read(final Kryo kryo, final Input input, final Class<Pair> pairClass) {
+ public <I extends InputShim> Pair read(KryoShim<I, ?> kryo, I input, Class<Pair> pairClass) {
return Pair.with(kryo.readClassAndObject(input), kryo.readClassAndObject(input));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
index ef105ce..1f41c0d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
@@ -67,4 +67,16 @@ public interface TypeRegistration<T> {
* @return the sole parameter
*/
Kryo registerWith(Kryo kryo);
+
+ /**
+ * Returns true if at least one of {@link #getShadedSerializer()}, {@link #getSerializerShim()}, or
+ * {@link #getFunctionOfShadedKryo()} is non null. Returns false if all are null.
+ *
+ * @return whether a serializer is defined for this type registration
+ */
+ default boolean hasSerializer() {
+ return null != getFunctionOfShadedKryo() ||
+ null != getSerializerShim() ||
+ null != getShadedSerializer();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
index 959605c..7783856 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
@@ -18,6 +18,8 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+import org.apache.commons.configuration.Configuration;
+
import java.io.InputStream;
import java.io.OutputStream;
@@ -80,4 +82,18 @@ public interface KryoShimService {
* @return this implementation's priority value
*/
int getPriority();
+
+ /**
+ * Attempt to incorporate the supplied configuration in future read/write calls.
+ * <p>
+ * This method is a wart that exists essentially just to support the old
+ * {@link HadoopPools#initialize(Configuration)} use-case.
+ * <p>
+ * This method is not guaranteed to have any effect on an instance of this interface
+ * after {@link #writeClassAndObject(Object, OutputStream)} or {@link #readClassAndObject(InputStream)}
+ * has been invoked on that particular instance.
+ *
+ * @param conf the configuration to apply to this service's internal serializer
+ */
+ void applyConfiguration(Configuration conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
index 9ccf2de..9184dd0 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.slf4j.Logger;
@@ -35,7 +36,9 @@ import java.util.ServiceLoader;
*/
public class KryoShimServiceLoader {
- private static volatile KryoShimService CACHED_SHIM_SERVICE;
+ private static volatile KryoShimService cachedShimService;
+
+ private static volatile Configuration conf;
private static final Logger log = LoggerFactory.getLogger(KryoShimServiceLoader.class);
@@ -46,6 +49,10 @@ public class KryoShimServiceLoader {
*/
public static final String SHIM_CLASS_SYSTEM_PROPERTY = "tinkerpop.kryo.shim";
+ public static void applyConfiguration(Configuration conf) {
+ KryoShimServiceLoader.conf = conf;
+ }
+
/**
* Return a reference to the shim service. This method may return a cached shim service
* unless {@code forceReload} is true. Calls to this method need not be externally
@@ -58,8 +65,8 @@ public class KryoShimServiceLoader {
*/
public static KryoShimService load(boolean forceReload) {
- if (null != CACHED_SHIM_SERVICE && !forceReload) {
- return CACHED_SHIM_SERVICE;
+ if (null != cachedShimService && !forceReload) {
+ return cachedShimService;
}
ArrayList<KryoShimService> services = new ArrayList<>();
@@ -109,7 +116,15 @@ public class KryoShimServiceLoader {
log.info("Set {} provider to {} ({}) because its priority value ({}) is the highest available",
KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
- return CACHED_SHIM_SERVICE = result;
+ Configuration userConf = conf;
+
+ if (null != userConf) {
+ log.info("Configuring {} provider {} with user-provided configuration",
+ KryoShimService.class.getSimpleName(), result);
+ result.applyConfiguration(userConf);
+ }
+
+ return cachedShimService = result;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
index 191cdd8..e5f9005 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
@@ -26,7 +26,7 @@ package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
*/
public interface SerializerShim<T> {
- <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, T starGraph);
+ <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, T object);
<I extends InputShim> T read(KryoShim<I, ?> kryo, I input, Class<T> clazz);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
index de1e2f9..06778e6 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +48,7 @@ public final class HadoopCombine extends Reducer<ObjectWritable, ObjectWritable,
@Override
public void setup(final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(context.getConfiguration());
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
this.mapReduce = MapReduce.createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.mapReduce.workerStart(MapReduce.Stage.COMBINE);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
index 9e6fac3..5fc7026 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
@@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,7 +51,7 @@ public final class HadoopMap extends Mapper<NullWritable, VertexWritable, Object
@Override
public void setup(final Mapper<NullWritable, VertexWritable, ObjectWritable, ObjectWritable>.Context context) {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(context.getConfiguration());
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
this.mapReduce = MapReduce.createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.mapReduce.workerStart(MapReduce.Stage.MAP);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
index 06dfba1..6ca7b8f 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,7 +48,7 @@ public final class HadoopReduce extends Reducer<ObjectWritable, ObjectWritable,
@Override
public void setup(final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(context.getConfiguration());
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
this.mapReduce = MapReduce.createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.mapReduce.workerStart(MapReduce.Stage.REDUCE);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
index c19b914..5753d90 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
@@ -18,7 +18,9 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;
@@ -66,4 +68,9 @@ public class HadoopPoolShimService implements KryoShimService {
public int getPriority() {
return 0;
}
+
+ @Override
+ public void applyConfiguration(Configuration conf) {
+ KryoShimServiceLoader.applyConfiguration(conf);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
index f3a1bac..0e5f135 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolsConfigurable.java
@@ -20,6 +20,8 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -28,7 +30,7 @@ public interface HadoopPoolsConfigurable extends Configurable {
@Override
public default void setConf(final Configuration configuration) {
- HadoopPools.initialize(configuration);
+ KryoShimServiceLoader.applyConfiguration(ConfUtil.makeApacheConfiguration(configuration));
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
index d7ed46b..a1daddf 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordReader.java
@@ -37,6 +37,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.VertexTerminator;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -71,7 +72,7 @@ public final class GryoRecordReader extends RecordReader<NullWritable, VertexWri
final Configuration configuration = context.getConfiguration();
if (configuration.get(Constants.GREMLIN_HADOOP_GRAPH_FILTER, null) != null)
this.graphFilter = VertexProgramHelper.deserialize(ConfUtil.makeApacheConfiguration(configuration), Constants.GREMLIN_HADOOP_GRAPH_FILTER);
- HadoopPools.initialize(configuration);
+ KryoShimServiceLoader.applyConfiguration(ConfUtil.makeApacheConfiguration(configuration));
this.gryoReader = HadoopPools.getGryoPool().takeReader();
long start = split.getStart();
final Path file = split.getPath();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
index 67a8339..2ea3394 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/GryoRecordWriter.java
@@ -25,8 +25,10 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -43,7 +45,7 @@ public final class GryoRecordWriter extends RecordWriter<NullWritable, VertexWri
public GryoRecordWriter(final DataOutputStream outputStream, final Configuration configuration) {
this.outputStream = outputStream;
this.hasEdges = configuration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, true);
- HadoopPools.initialize(configuration);
+ KryoShimServiceLoader.applyConfiguration(ConfUtil.makeApacheConfiguration(configuration));
this.gryoWriter = HadoopPools.getGryoPool().takeWriter();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index c2b85dd..9e5ac53 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -38,6 +38,7 @@ import org.apache.tinkerpop.gremlin.spark.process.computer.payload.Payload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.structure.util.Attachable;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
@@ -89,7 +90,7 @@ public final class SparkExecutor {
graphRDD.leftOuterJoin(viewIncomingRDD)) // every other iteration may have views and messages
// for each partition of vertices emit a view and their outgoing messages
.mapPartitionsToPair(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
final String[] vertexComputeKeysArray = VertexProgramHelper.vertexComputeKeysAsArray(workerVertexProgram.getVertexComputeKeys()); // the compute keys as an array
final SparkMessenger<M> messenger = new SparkMessenger<>();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java
deleted file mode 100644
index 4c99e70..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/TinkerPopKryoRegistrator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.spark.structure.io;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.Serializer;
-import com.esotericsoftware.kryo.serializers.JavaSerializer;
-import com.google.common.base.Preconditions;
-import org.apache.spark.serializer.KryoRegistrator;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroupStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderGlobalStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalExplanation;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.ObjectWritableSerializer;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.VertexWritableSerializer;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedSerializerAdapter;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.TypeRegistration;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * A spark.kryo.registrator implementation that installs TinkerPop types.
- * This is intended for use with spark.serializer=KryoSerializer, not GryoSerializer.
- */
-public class TinkerPopKryoRegistrator implements KryoRegistrator {
-
- private static final Logger log = LoggerFactory.getLogger(TinkerPopKryoRegistrator.class);
-
- @Override
- public void registerClasses(Kryo kryo) {
- // TinkerPop type registrations copied from GyroSerializer's constructor
- kryo.register(MessagePayload.class);
- kryo.register(ViewIncomingPayload.class);
- kryo.register(ViewOutgoingPayload.class);
- kryo.register(ViewPayload.class);
- kryo.register(VertexWritable.class, new UnshadedSerializerAdapter<>(new VertexWritableSerializer()));
- kryo.register(ObjectWritable.class, new UnshadedSerializerAdapter<>(new ObjectWritableSerializer<>()));
-
- Set<Class<?>> shimmedClasses = new HashSet<>();
-
- Set<Class<?>> javaSerializationClasses = new HashSet<>();
-
- // Copy GryoMapper's default registrations
- for (TypeRegistration<?> tr : GryoMapper.build().create().getTypeRegistrations()) {
- // Special case for JavaSerializer, which is generally implemented in terms of TinkerPop's
- // problematic static GryoMapper/GryoSerializer pool (these are handled below the loop)
- org.apache.tinkerpop.shaded.kryo.Serializer<?> shadedSerializer = tr.getShadedSerializer();
- SerializerShim<?> serializerShim = tr.getSerializerShim();
- if (null != shadedSerializer &&
- shadedSerializer.getClass().equals(org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer.class)) {
- javaSerializationClasses.add(tr.getTargetClass());
- } else if (null != serializerShim) {
- log.debug("Registering class {} to serializer shim {} (serializer shim class {})",
- tr.getTargetClass(), serializerShim, serializerShim.getClass());
- kryo.register(tr.getTargetClass(), new UnshadedSerializerAdapter<>(serializerShim));
- shimmedClasses.add(tr.getTargetClass());
- } else {
- // Register with the default behavior (FieldSerializer)
- log.debug("Registering class {} with default serializer", tr.getTargetClass());
- kryo.register(tr.getTargetClass());
- }
- }
-
- Map<Class<?>, Serializer<?>> javaSerializerReplacements = new HashMap<>();
- javaSerializerReplacements.put(GroupStep.GroupBiOperator.class, new JavaSerializer());
- javaSerializerReplacements.put(OrderGlobalStep.OrderBiOperator.class, null);
- javaSerializerReplacements.put(TraversalExplanation.class, null);
-
- for (Map.Entry<Class<?>, Serializer<?>> e : javaSerializerReplacements.entrySet()) {
- Class<?> c = e.getKey();
- Serializer<?> s = e.getValue();
-
- if (javaSerializationClasses.remove(c)) {
- if (null != s) {
- log.debug("Registering class {} with serializer {}", c, s);
- kryo.register(c, s);
- } else {
- log.debug("Registering class {} with default serializer", c);
- kryo.register(c);
- }
- } else {
- log.debug("Registering class {} with JavaSerializer", c);
- kryo.register(c, new JavaSerializer());
- }
- }
-
- // We really care about StarGraph's shim serializer, so make sure we registered it
- if (!shimmedClasses.contains(StarGraph.class)) {
- log.warn("No SerializerShim found for StarGraph");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
index 2c1dfa2..28a4d55 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
@@ -78,7 +78,7 @@ public final class GryoSerializer extends Serializer {
}
}
this.gryoPool = GryoPool.build().
- poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, 256)).
+ poolSize(sparkConfiguration.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE, GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT)).
ioRegistries(makeApacheConfiguration(sparkConfiguration).getList(GryoPool.CONFIG_IO_REGISTRY, Collections.emptyList())).
initializeMapper(builder -> {
try {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java
new file mode 100644
index 0000000..8b21e21
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Copyright DataStax, Inc.
+ * <p>
+ * Please see the included license file for details.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import org.apache.spark.SparkConf;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
+import org.javatuples.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+/**
+ * A {@link KryoSerializer} that attempts to honor {@link GryoPool#CONFIG_IO_REGISTRY}.
+ */
+public class IoRegistryAwareKryoSerializer extends KryoSerializer {
+
+ private final SparkConf conf;
+
+ private static final Logger log = LoggerFactory.getLogger(IoRegistryAwareKryoSerializer.class);
+
+ public IoRegistryAwareKryoSerializer(SparkConf conf) {
+ super(conf);
+ // store conf so that we can access its registry (if one is present) in newKryo()
+ this.conf = conf;
+ }
+
+ @Override
+ public Kryo newKryo() {
+ Kryo kryo = super.newKryo();
+
+ return applyIoRegistryIfPresent(kryo);
+ }
+
+ private Kryo applyIoRegistryIfPresent(Kryo kryo) {
+ if (!conf.contains(GryoPool.CONFIG_IO_REGISTRY)) {
+ log.info("SparkConf {} does not contain setting {}, skipping {} handling",
+ GryoPool.CONFIG_IO_REGISTRY, conf, IoRegistry.class.getCanonicalName());
+ return kryo;
+ }
+
+ String registryClassnames = conf.get(GryoPool.CONFIG_IO_REGISTRY);
+
+ for (String registryClassname : registryClassnames.split(",")) {
+ final IoRegistry registry;
+
+ try {
+ registry = (IoRegistry) Class.forName(registryClassname).newInstance();
+ log.info("Instantiated {}", registryClassname);
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ log.error("Unable to reflectively instantiate the {} implementation named {}",
+ IoRegistry.class.getCanonicalName(), registryClassname, e);
+ return kryo;
+ }
+
+ // Left is the class targeted for serialization, right is a mess of potential types, including
+ // a shaded Serializer impl, unshaded Serializer impl, or Function<shaded.Kryo,shaded.Serializer>
+ final List<Pair<Class, Object>> serializers = registry.find(GryoIo.class);
+
+ if (null == serializers) {
+ log.info("Invoking find({}.class) returned null on registry {}; ignoring this registry",
+ GryoIo.class.getCanonicalName(), registry);
+ return kryo;
+ }
+
+ for (Pair<Class, Object> p : serializers) {
+ if (null == p.getValue1()) {
+ // null on the right is fine
+ log.info("Registering {} with default serializer", p.getValue0());
+ kryo.register(p.getValue0());
+ } else if (p.getValue1() instanceof Serializer) {
+ // unshaded serializer on the right is fine
+ log.info("Registering {} with serializer {}", p.getValue0(), p.getValue1());
+ kryo.register(p.getValue0(), (Serializer) p.getValue1());
+ } else {
+ // anything else on the right is unsupported with Spark
+ log.error("Serializer {} found in {} must implement {} " +
+ "(the shaded interface {} is not supported on Spark). This class will be registered with " +
+ "the default behavior of Spark's KryoSerializer.",
+ p.getValue1(), registryClassname, Serializer.class.getCanonicalName(),
+ org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
+ kryo.register(p.getValue0());
+ }
+ }
+ }
+
+ return kryo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java
new file mode 100644
index 0000000..bdb80fd
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.serializers.JavaSerializer;
+import org.apache.spark.serializer.KryoRegistrator;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedSerializerAdapter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.TypeRegistration;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * A spark.kryo.registrator implementation that installs TinkerPop types.
+ * This is intended for use with spark.serializer=KryoSerializer, not GryoSerializer.
+ */
+public class TinkerPopKryoRegistrator implements KryoRegistrator {
+
+ private static final Logger log = LoggerFactory.getLogger(TinkerPopKryoRegistrator.class);
+
+ @Override
+ public void registerClasses(Kryo kryo) {
+ registerClasses(kryo, Collections.emptyMap(), Collections.emptySet());
+ }
+
+ /**
+ * Register TinkerPop's classes with the supplied {@link Kryo} instance
+ * while honoring optional overrides and optional class blacklist ("blackset"?).
+ *
+ * @param kryo the Kryo serializer instance with which to register types
+ * @param serializerOverrides serializer mappings that override this class's defaults
+ * @param blacklist classes which should not be registered at all, even if there is an override entry
+ * or if they would be registered by this class by default (does not affect Kryo's
+ * built-in registrations, e.g. String.class).
+ */
+ public void registerClasses(Kryo kryo, Map<Class<?>, Serializer<?>> serializerOverrides, Set<Class<?>> blacklist) {
+ // Apply TinkerPop type registrations copied from GyroSerializer's constructor
+ for (Map.Entry<Class<?>, Serializer<?>> ent : getExtraRegistrations().entrySet()) {
+ Class<?> targetClass = ent.getKey();
+ Serializer<?> ser = ent.getValue();
+
+ // Is this class blacklisted? Skip it. (takes precedence over serializerOverrides)
+ if (blacklist.contains(targetClass)) {
+ log.debug("Not registering serializer for {} (blacklisted)", targetClass);
+ continue;
+ }
+
+ if (checkForAndApplySerializerOverride(serializerOverrides, kryo, targetClass)) {
+ // do nothing but skip the remaining else(-if) clauses
+ } else if (null == ser) {
+ log.debug("Registering {} with default serializer", targetClass);
+ kryo.register(targetClass);
+ } else {
+ log.debug("Registering {} with serializer {}", targetClass, ser);
+ kryo.register(targetClass, ser);
+ }
+ }
+
+ Set<Class<?>> shimmedClassesFromGryoMapper = new HashSet<>();
+
+ // Apply GryoMapper's default registrations
+ for (TypeRegistration<?> tr : GryoMapper.build().create().getTypeRegistrations()) {
+ // Is this class blacklisted? Skip it. (takes precedence over serializerOverrides)
+ if (blacklist.contains(tr.getTargetClass())) {
+ log.debug("Not registering serializer for {} (blacklisted)", tr.getTargetClass());
+ continue;
+ }
+
+ final org.apache.tinkerpop.shaded.kryo.Serializer<?> shadedSerializer = tr.getShadedSerializer();
+ final SerializerShim<?> serializerShim = tr.getSerializerShim();
+ final java.util.function.Function<
+ org.apache.tinkerpop.shaded.kryo.Kryo,
+ org.apache.tinkerpop.shaded.kryo.Serializer> functionOfShadedKryo = tr.getFunctionOfShadedKryo();
+
+ // Apply overrides with the highest case-precedence
+ if (checkForAndApplySerializerOverride(serializerOverrides, kryo, tr.getTargetClass())) {
+ // do nothing but skip the remaining else(-if) clauses
+ } else if (null != shadedSerializer) {
+ if (shadedSerializer.getClass().equals(org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer.class)) {
+ // Convert GryoMapper's shaded JavaSerializer mappings to their unshaded equivalents
+ log.debug("Registering {} with JavaSerializer", tr.getTargetClass());
+ kryo.register(tr.getTargetClass(), new JavaSerializer());
+ } else {
+ // There's supposed to be a check in GryoMapper that prevents this from happening
+ log.error("GryoMapper's default serialization registration for {} is a {}. " +
+ "This is probably a bug in TinkerPop (this is not a valid default registration). " +
+ "I am configuring Spark to use Kryo's default serializer for this class, " +
+ "but this may cause serialization failures at runtime.",
+ tr.getTargetClass(),
+ org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
+ kryo.register(tr.getTargetClass());
+ }
+ } else if (null != serializerShim) {
+ // Wrap shim serializers in an adapter for Spark's unshaded Kryo
+ log.debug("Registering {} to serializer shim {} (serializer shim {})",
+ tr.getTargetClass(), serializerShim, serializerShim.getClass());
+ kryo.register(tr.getTargetClass(), new UnshadedSerializerAdapter<>(serializerShim));
+ shimmedClassesFromGryoMapper.add(tr.getTargetClass());
+ } else if (null != functionOfShadedKryo) {
+ // As with shaded serializers, there's supposed to be a check in GryoMapper that prevents this from happening
+ log.error("GryoMapper's default serialization registration for {} is a Function<{},{}>. " +
+ "This is probably a bug in TinkerPop (this is not a valid default registration). " +
+ "I am configuring Spark to use Kryo's default serializer instead of this function, " +
+ "but this may cause serialization failures at runtime.",
+ tr.getTargetClass(),
+ org.apache.tinkerpop.shaded.kryo.Kryo.class.getCanonicalName(),
+ org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
+ kryo.register(tr.getTargetClass());
+ } else {
+ // Register all other classes with the default behavior (FieldSerializer)
+ log.debug("Registering {} with default serializer", tr.getTargetClass());
+ kryo.register(tr.getTargetClass());
+ }
+ }
+
+ // StarGraph's shim serializer is especially important on Spark for efficiency reasons,
+ // so log a warning if we failed to register it somehow
+ if (!shimmedClassesFromGryoMapper.contains(StarGraph.class)) {
+ log.warn("No SerializerShim found for StarGraph");
+ }
+ }
+
+ private LinkedHashMap<Class<?>, Serializer<?>> getExtraRegistrations() {
+
+ /* The map returned by this method MUST have a fixed iteration order!
+ *
+ * The order itself is irrelevant, so long as it is completely stable at runtime.
+ *
+ * LinkedHashMap satisfies this requirement (its contract specifies
+ * iteration in key-insertion-order).
+ */
+
+ LinkedHashMap<Class<?>, Serializer<?>> m = new LinkedHashMap<>();
+ // The following entries were copied from GryoSerializer's constructor
+ // This could be turned into a static collection on GryoSerializer to avoid
+ // duplication, but it would be a bit cumbersome to do so without disturbing
+ // the ordering of the existing entries in that constructor, since not all
+ // of the entries are for TinkerPop (and the ordering is significant).
+ m.put(MessagePayload.class, null);
+ m.put(ViewIncomingPayload.class, null);
+ m.put(ViewOutgoingPayload.class, null);
+ m.put(ViewPayload.class, null);
+ m.put(VertexWritable.class, new UnshadedSerializerAdapter<>(new VertexWritableSerializer()));
+ m.put(ObjectWritable.class, new UnshadedSerializerAdapter<>(new ObjectWritableSerializer<>()));
+
+ return m;
+ }
+
+ private boolean checkForAndApplySerializerOverride(Map<Class<?>, Serializer<?>> serializerOverrides,
+ Kryo kryo, Class<?> targetClass) {
+ if (serializerOverrides.containsKey(targetClass)) {
+ Serializer<?> ser = serializerOverrides.get(targetClass);
+ if (null == ser) {
+ // null means use Kryo's default serializer
+ log.debug("Registering {} with default serializer per overrides", targetClass);
+ kryo.register(targetClass);
+ } else {
+ // nonnull means use that serializer
+ log.debug("Registering {} with serializer {} per overrides", targetClass, ser);
+ kryo.register(targetClass, ser);
+ }
+ return true;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
index d0411e8..a524a97 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
@@ -24,92 +24,131 @@
*/
package org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded;
-import com.twitter.chill.KryoInstantiator;
-import com.twitter.chill.KryoPool;
-import com.twitter.chill.SerDeState;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
import org.apache.spark.SparkConf;
-import org.apache.spark.serializer.KryoSerializer;
-import org.apache.tinkerpop.gremlin.spark.structure.io.TinkerPopKryoRegistrator;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.IoRegistryAwareKryoSerializer;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.concurrent.LinkedBlockingQueue;
public class UnshadedKryoShimService implements KryoShimService {
- public static final String SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY = "tinkerpop.kryo.poolsize";
-
private static final Logger log = LoggerFactory.getLogger(UnshadedKryoShimService.class);
- private static final int SPARK_KRYO_POOL_SIZE_DEFAULT = 8;
- private final KryoSerializer sparkKryoSerializer;
- private final KryoPool kryoPool;
+ private static final LinkedBlockingQueue<Kryo> KRYOS = new LinkedBlockingQueue<>();
- public UnshadedKryoShimService() {
- this(TinkerPopKryoRegistrator.class.getCanonicalName(), getDefaultKryoPoolSize());
- }
+ private static volatile boolean initialized;
- public UnshadedKryoShimService(String sparkKryoRegistratorClassname, int kryoPoolSize) {
- SparkConf sparkConf = new SparkConf();
- sparkConf.set("spark.serializer", KryoSerializer.class.getCanonicalName());
- sparkConf.set("spark.kryo.registrator", sparkKryoRegistratorClassname);
- sparkKryoSerializer = new KryoSerializer(sparkConf);
- kryoPool = KryoPool.withByteArrayOutputStream(kryoPoolSize, new KryoInstantiator());
- }
+ public UnshadedKryoShimService() { }
@Override
public Object readClassAndObject(InputStream source) {
- SerDeState sds = null;
- try {
- sds = kryoPool.borrow();
- sds.setInput(source);
+ LinkedBlockingQueue<Kryo> kryos = initialize();
- return sds.readClassAndObject();
+ Kryo k = null;
+ try {
+ k = kryos.take();
+
+ return k.readClassAndObject(new Input(source));
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
} finally {
- kryoPool.release(sds);
+ try {
+ kryos.put(k);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
}
}
@Override
public void writeClassAndObject(Object o, OutputStream sink) {
- SerDeState sds = null;
+
+ LinkedBlockingQueue<Kryo> kryos = initialize();
+
+ Kryo k = null;
try {
- sds = kryoPool.borrow();
+ k = kryos.take();
- sds.writeClassAndObject(o); // this writes to an internal buffer
+ Output kryoOutput = new Output(sink);
- sds.writeOutputTo(sink); // this copies the internal buffer to sink
+ k.writeClassAndObject(kryoOutput, o);
- sink.flush();
- } catch (IOException e) {
+ kryoOutput.flush();
+ } catch (InterruptedException e) {
throw new RuntimeException(e);
} finally {
- kryoPool.release(sds);
+ try {
+ kryos.put(k);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
}
}
@Override
public int getPriority() {
- return 1024;
+ return 50;
}
- private static int getDefaultKryoPoolSize() {
- String raw = System.getProperty(SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY);
+ @Override
+ public void applyConfiguration(Configuration conf) {
+ initialize(conf);
+ }
- int size = SPARK_KRYO_POOL_SIZE_DEFAULT;
- try {
- size = Integer.valueOf(raw);
- log.info("Setting kryo pool size to {} according to system property {}", size,
- SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY);
- } catch (NumberFormatException e) {
- log.error("System property {}={} could not be parsed as an integer, using default value {}",
- SPARK_KRYO_POOL_SIZE_SYSTEM_PROPERTY, raw, size, e);
+ private LinkedBlockingQueue<Kryo> initialize() {
+ return initialize(new BaseConfiguration());
+ }
+
+ private LinkedBlockingQueue<Kryo> initialize(Configuration conf) {
+ // DCL is safe in this case due to volatility
+ if (!initialized) {
+ synchronized (UnshadedKryoShimService.class) {
+ if (!initialized) {
+ SparkConf sparkConf = new SparkConf();
+
+ // Copy the user's IoRegistry from the param conf to the SparkConf we just created
+ String regStr = conf.getString(GryoPool.CONFIG_IO_REGISTRY);
+ if (null != regStr) { // SparkConf rejects null values with NPE, so this has to be checked before set(...)
+ sparkConf.set(GryoPool.CONFIG_IO_REGISTRY, regStr);
+ }
+ // Setting spark.serializer here almost certainly isn't necessary, but it doesn't hurt
+ sparkConf.set("spark.serializer", IoRegistryAwareKryoSerializer.class.getCanonicalName());
+
+ String registrator = conf.getString("spark.kryo.registrator");
+ if (null != registrator) {
+ sparkConf.set("spark.kryo.registrator", registrator);
+ log.info("Copied spark.kryo.registrator: {}", registrator);
+ } else {
+ log.info("Not copying spark.kryo.registrator");
+ }
+
+ // Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf)
+ int poolSize = conf.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
+ GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
+ // Instantiate the spark.serializer
+ final IoRegistryAwareKryoSerializer ioReg = new IoRegistryAwareKryoSerializer(sparkConf);
+ // Setup a pool backed by our spark.serializer instance
+
+ for (int i = 0; i < poolSize; i++) {
+ KRYOS.add(ioReg.newKryo());
+ }
+
+ initialized = true;
+ }
+ }
}
- return size;
+ return KRYOS;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService b/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService
new file mode 100644
index 0000000..68712a6
--- /dev/null
+++ b/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService
@@ -0,0 +1 @@
+org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService # Supports Spark
[15/18] incubator-tinkerpop git commit: split up the GryoRegistrator
and GryoSerializer based tests into two different full runs of
ProcessComputerSuite. The former is an Integration test.
Posted by ok...@apache.org.
split up the GryoRegistrator and GryoSerializer based tests into two different full runs of ProcessComputerSuite. The former is an Integration test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/293fe29c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/293fe29c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/293fe29c
Branch: refs/heads/master
Commit: 293fe29c6735952a7a6b0ed4814eea72adfdde66
Parents: 08040f8
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Jun 6 20:59:21 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Jun 6 20:59:21 2016 -0600
----------------------------------------------------------------------
.../gremlin/spark/AbstractSparkTest.java | 4 ++
...tratorGraphComputerProcessIntegrateTest.java | 33 ++++++++++++
...SparkHadoopGraphGryoRegistratorProvider.java | 56 ++++++++++++++++++++
.../computer/SparkHadoopGraphProvider.java | 19 ++-----
4 files changed, 97 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
index 6d2231f..4d584ff 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
@@ -26,6 +26,7 @@ import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService;
import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -34,6 +35,8 @@ import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader.SHIM_CLASS_SYSTEM_PROPERTY;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@@ -43,6 +46,7 @@ public abstract class AbstractSparkTest {
@After
@Before
public void setupTest() {
+ System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY);
SparkConf sparkConfiguration = new SparkConf();
sparkConfiguration.setAppName(this.getClass().getCanonicalName() + "-setupTest");
sparkConfiguration.set("spark.master", "local[4]");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java
new file mode 100644
index 0000000..29f627d
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGryoRegistratorGraphComputerProcessIntegrateTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(ProcessComputerSuite.class)
+@GraphProviderClass(provider = SparkHadoopGraphGryoRegistratorProvider.class, graph = HadoopGraph.class)
+public class SparkGryoRegistratorGraphComputerProcessIntegrateTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
new file mode 100644
index 0000000..df36a13
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.spark.structure.Spark;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
+
+import java.util.Map;
+
+import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader.SHIM_CLASS_SYSTEM_PROPERTY;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkHadoopGraphGryoRegistratorProvider extends SparkHadoopGraphProvider {
+
+ private static boolean firstTest = true;
+
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
+ config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false); // ensure the context doesn't stay open for the GryoSerializer tests
+ config.put("spark.serializer", KryoSerializer.class.getCanonicalName());
+ config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName());
+ //
+ if (firstTest) {
+ firstTest = false;
+ Spark.close();
+ System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, UnshadedKryoShimService.class.getCanonicalName());
+ KryoShimServiceLoader.load(true);
+ }
+ //
+ return config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/293fe29c/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index b1f610d..1fc29cf 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -55,13 +55,12 @@ import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimSe
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
-public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
+public class SparkHadoopGraphProvider extends HadoopGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT,
- !test.equals(ProgramTest.Traversals.class) && !test.equals(GroovyProgramTest.class)); // this makes the test suite go really fast
+ config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
// toy graph inputRDD does not have corresponding outputRDD so where jobs chain, it fails (failing makes sense)
if (null != loadGraphWith &&
@@ -89,20 +88,10 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
SugarTestHelper.clearRegistry(this);
}
+ System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY);
config.put(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
-
-
config.put("spark.master", "local[4]");
- if (RANDOM.nextBoolean()) {
- System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, HadoopPoolShimService.class.getCanonicalName());
- KryoShimServiceLoader.load(true);
- config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
- } else {
- System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, UnshadedKryoShimService.class.getCanonicalName());
- KryoShimServiceLoader.load(true);
- config.put("spark.serializer", KryoSerializer.class.getCanonicalName());
- config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName());
- }
+ config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
config.put("spark.kryo.registrationRequired", true);
return config;
}
[08/18] incubator-tinkerpop git commit: TINKERPOP-1321 Shim change
and test tweaks
Posted by ok...@apache.org.
TINKERPOP-1321 Shim change and test tweaks
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/88ac3047
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/88ac3047
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/88ac3047
Branch: refs/heads/master
Commit: 88ac30472cac2c7ca7c604de5de85e6709791611
Parents: ebd2f85
Author: Dan LaRocque <da...@hopcount.org>
Authored: Mon Jun 6 15:32:37 2016 -0400
Committer: Dan LaRocque <da...@hopcount.org>
Committed: Mon Jun 6 15:34:43 2016 -0400
----------------------------------------------------------------------
.../structure/io/gryo/GryoRegistrator.java | 194 +++++++++++++++++++
.../io/gryo/TinkerPopKryoRegistrator.java | 194 -------------------
.../unshaded/UnshadedKryoShimService.java | 2 +-
.../computer/SparkHadoopGraphProvider.java | 11 +-
4 files changed, 204 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/88ac3047/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
new file mode 100644
index 0000000..1ae8c5c
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.serializers.JavaSerializer;
+import org.apache.spark.serializer.KryoRegistrator;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedSerializerAdapter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.TypeRegistration;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+/**
+ * A spark.kryo.registrator implementation that installs TinkerPop types.
+ * This is intended for use with spark.serializer=KryoSerializer, not GryoSerializer.
+ */
+public class GryoRegistrator implements KryoRegistrator {
+
+ private static final Logger log = LoggerFactory.getLogger(GryoRegistrator.class);
+
+ @Override
+ public void registerClasses(Kryo kryo) {
+ registerClasses(kryo, Collections.emptyMap(), Collections.emptySet());
+ }
+
+ /**
+ * Register TinkerPop's classes with the supplied {@link Kryo} instance
+ * while honoring optional overrides and optional class blacklist ("blackset"?).
+ *
+ * @param kryo the Kryo serializer instance with which to register types
+ * @param serializerOverrides serializer mappings that override this class's defaults
+ * @param blacklist classes which should not be registered at all, even if there is an override entry
+ * or if they would be registered by this class by default (does not affect Kryo's
+ * built-in registrations, e.g. String.class).
+ */
+ public void registerClasses(Kryo kryo, Map<Class<?>, Serializer<?>> serializerOverrides, Set<Class<?>> blacklist) {
+ // Apply TinkerPop type registrations copied from GyroSerializer's constructor
+ for (Map.Entry<Class<?>, Serializer<?>> ent : getExtraRegistrations().entrySet()) {
+ Class<?> targetClass = ent.getKey();
+ Serializer<?> ser = ent.getValue();
+
+ // Is this class blacklisted? Skip it. (takes precedence over serializerOverrides)
+ if (blacklist.contains(targetClass)) {
+ log.debug("Not registering serializer for {} (blacklisted)", targetClass);
+ continue;
+ }
+
+ if (checkForAndApplySerializerOverride(serializerOverrides, kryo, targetClass)) {
+ // do nothing but skip the remaining else(-if) clauses
+ } else if (null == ser) {
+ log.debug("Registering {} with default serializer", targetClass);
+ kryo.register(targetClass);
+ } else {
+ log.debug("Registering {} with serializer {}", targetClass, ser);
+ kryo.register(targetClass, ser);
+ }
+ }
+
+ Set<Class<?>> shimmedClassesFromGryoMapper = new HashSet<>();
+
+ // Apply GryoMapper's default registrations
+ for (TypeRegistration<?> tr : GryoMapper.build().create().getTypeRegistrations()) {
+ // Is this class blacklisted? Skip it. (takes precedence over serializerOverrides)
+ if (blacklist.contains(tr.getTargetClass())) {
+ log.debug("Not registering serializer for {} (blacklisted)", tr.getTargetClass());
+ continue;
+ }
+
+ final org.apache.tinkerpop.shaded.kryo.Serializer<?> shadedSerializer = tr.getShadedSerializer();
+ final SerializerShim<?> serializerShim = tr.getSerializerShim();
+ final java.util.function.Function<
+ org.apache.tinkerpop.shaded.kryo.Kryo,
+ org.apache.tinkerpop.shaded.kryo.Serializer> functionOfShadedKryo = tr.getFunctionOfShadedKryo();
+
+ // Apply overrides with the highest case-precedence
+ if (checkForAndApplySerializerOverride(serializerOverrides, kryo, tr.getTargetClass())) {
+ // do nothing but skip the remaining else(-if) clauses
+ } else if (null != shadedSerializer) {
+ if (shadedSerializer.getClass().equals(org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer.class)) {
+ // Convert GryoMapper's shaded JavaSerializer mappings to their unshaded equivalents
+ log.debug("Registering {} with JavaSerializer", tr.getTargetClass());
+ kryo.register(tr.getTargetClass(), new JavaSerializer());
+ } else {
+ // There's supposed to be a check in GryoMapper that prevents this from happening
+ log.error("GryoMapper's default serialization registration for {} is a {}. " +
+ "This is probably a bug in TinkerPop (this is not a valid default registration). " +
+ "I am configuring Spark to use Kryo's default serializer for this class, " +
+ "but this may cause serialization failures at runtime.",
+ tr.getTargetClass(),
+ org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
+ kryo.register(tr.getTargetClass());
+ }
+ } else if (null != serializerShim) {
+ // Wrap shim serializers in an adapter for Spark's unshaded Kryo
+ log.debug("Registering {} to serializer shim {} (serializer shim {})",
+ tr.getTargetClass(), serializerShim, serializerShim.getClass());
+ kryo.register(tr.getTargetClass(), new UnshadedSerializerAdapter<>(serializerShim));
+ shimmedClassesFromGryoMapper.add(tr.getTargetClass());
+ } else if (null != functionOfShadedKryo) {
+ // As with shaded serializers, there's supposed to be a check in GryoMapper that prevents this from happening
+ log.error("GryoMapper's default serialization registration for {} is a Function<{},{}>. " +
+ "This is probably a bug in TinkerPop (this is not a valid default registration). " +
+ "I am configuring Spark to use Kryo's default serializer instead of this function, " +
+ "but this may cause serialization failures at runtime.",
+ tr.getTargetClass(),
+ org.apache.tinkerpop.shaded.kryo.Kryo.class.getCanonicalName(),
+ org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
+ kryo.register(tr.getTargetClass());
+ } else {
+ // Register all other classes with the default behavior (FieldSerializer)
+ log.debug("Registering {} with default serializer", tr.getTargetClass());
+ kryo.register(tr.getTargetClass());
+ }
+ }
+
+ // StarGraph's shim serializer is especially important on Spark for efficiency reasons,
+ // so log a warning if we failed to register it somehow
+ if (!shimmedClassesFromGryoMapper.contains(StarGraph.class)) {
+ log.warn("No SerializerShim found for StarGraph");
+ }
+ }
+
+ private LinkedHashMap<Class<?>, Serializer<?>> getExtraRegistrations() {
+
+ /* The map returned by this method MUST have a fixed iteration order!
+ *
+ * The order itself is irrelevant, so long as it is completely stable at runtime.
+ *
+ * LinkedHashMap satisfies this requirement (its contract specifies
+ * iteration in key-insertion-order).
+ */
+
+ LinkedHashMap<Class<?>, Serializer<?>> m = new LinkedHashMap<>();
+ // The following entries were copied from GryoSerializer's constructor
+ // This could be turned into a static collection on GryoSerializer to avoid
+ // duplication, but it would be a bit cumbersome to do so without disturbing
+ // the ordering of the existing entries in that constructor, since not all
+ // of the entries are for TinkerPop (and the ordering is significant).
+ m.put(MessagePayload.class, null);
+ m.put(ViewIncomingPayload.class, null);
+ m.put(ViewOutgoingPayload.class, null);
+ m.put(ViewPayload.class, null);
+ m.put(VertexWritable.class, new UnshadedSerializerAdapter<>(new VertexWritableSerializer()));
+ m.put(ObjectWritable.class, new UnshadedSerializerAdapter<>(new ObjectWritableSerializer<>()));
+
+ return m;
+ }
+
+ private boolean checkForAndApplySerializerOverride(Map<Class<?>, Serializer<?>> serializerOverrides,
+ Kryo kryo, Class<?> targetClass) {
+ if (serializerOverrides.containsKey(targetClass)) {
+ Serializer<?> ser = serializerOverrides.get(targetClass);
+ if (null == ser) {
+ // null means use Kryo's default serializer
+ log.debug("Registering {} with default serializer per overrides", targetClass);
+ kryo.register(targetClass);
+ } else {
+ // nonnull means use that serializer
+ log.debug("Registering {} with serializer {} per overrides", targetClass, ser);
+ kryo.register(targetClass, ser);
+ }
+ return true;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/88ac3047/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java
deleted file mode 100644
index bdb80fd..0000000
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/TinkerPopKryoRegistrator.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.spark.structure.io.gryo;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.Serializer;
-import com.esotericsoftware.kryo.serializers.JavaSerializer;
-import org.apache.spark.serializer.KryoRegistrator;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
-import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
-import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedSerializerAdapter;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.TypeRegistration;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-/**
- * A spark.kryo.registrator implementation that installs TinkerPop types.
- * This is intended for use with spark.serializer=KryoSerializer, not GryoSerializer.
- */
-public class TinkerPopKryoRegistrator implements KryoRegistrator {
-
- private static final Logger log = LoggerFactory.getLogger(TinkerPopKryoRegistrator.class);
-
- @Override
- public void registerClasses(Kryo kryo) {
- registerClasses(kryo, Collections.emptyMap(), Collections.emptySet());
- }
-
- /**
- * Register TinkerPop's classes with the supplied {@link Kryo} instance
- * while honoring optional overrides and optional class blacklist ("blackset"?).
- *
- * @param kryo the Kryo serializer instance with which to register types
- * @param serializerOverrides serializer mappings that override this class's defaults
- * @param blacklist classes which should not be registered at all, even if there is an override entry
- * or if they would be registered by this class by default (does not affect Kryo's
- * built-in registrations, e.g. String.class).
- */
- public void registerClasses(Kryo kryo, Map<Class<?>, Serializer<?>> serializerOverrides, Set<Class<?>> blacklist) {
- // Apply TinkerPop type registrations copied from GyroSerializer's constructor
- for (Map.Entry<Class<?>, Serializer<?>> ent : getExtraRegistrations().entrySet()) {
- Class<?> targetClass = ent.getKey();
- Serializer<?> ser = ent.getValue();
-
- // Is this class blacklisted? Skip it. (takes precedence over serializerOverrides)
- if (blacklist.contains(targetClass)) {
- log.debug("Not registering serializer for {} (blacklisted)", targetClass);
- continue;
- }
-
- if (checkForAndApplySerializerOverride(serializerOverrides, kryo, targetClass)) {
- // do nothing but skip the remaining else(-if) clauses
- } else if (null == ser) {
- log.debug("Registering {} with default serializer", targetClass);
- kryo.register(targetClass);
- } else {
- log.debug("Registering {} with serializer {}", targetClass, ser);
- kryo.register(targetClass, ser);
- }
- }
-
- Set<Class<?>> shimmedClassesFromGryoMapper = new HashSet<>();
-
- // Apply GryoMapper's default registrations
- for (TypeRegistration<?> tr : GryoMapper.build().create().getTypeRegistrations()) {
- // Is this class blacklisted? Skip it. (takes precedence over serializerOverrides)
- if (blacklist.contains(tr.getTargetClass())) {
- log.debug("Not registering serializer for {} (blacklisted)", tr.getTargetClass());
- continue;
- }
-
- final org.apache.tinkerpop.shaded.kryo.Serializer<?> shadedSerializer = tr.getShadedSerializer();
- final SerializerShim<?> serializerShim = tr.getSerializerShim();
- final java.util.function.Function<
- org.apache.tinkerpop.shaded.kryo.Kryo,
- org.apache.tinkerpop.shaded.kryo.Serializer> functionOfShadedKryo = tr.getFunctionOfShadedKryo();
-
- // Apply overrides with the highest case-precedence
- if (checkForAndApplySerializerOverride(serializerOverrides, kryo, tr.getTargetClass())) {
- // do nothing but skip the remaining else(-if) clauses
- } else if (null != shadedSerializer) {
- if (shadedSerializer.getClass().equals(org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer.class)) {
- // Convert GryoMapper's shaded JavaSerializer mappings to their unshaded equivalents
- log.debug("Registering {} with JavaSerializer", tr.getTargetClass());
- kryo.register(tr.getTargetClass(), new JavaSerializer());
- } else {
- // There's supposed to be a check in GryoMapper that prevents this from happening
- log.error("GryoMapper's default serialization registration for {} is a {}. " +
- "This is probably a bug in TinkerPop (this is not a valid default registration). " +
- "I am configuring Spark to use Kryo's default serializer for this class, " +
- "but this may cause serialization failures at runtime.",
- tr.getTargetClass(),
- org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
- kryo.register(tr.getTargetClass());
- }
- } else if (null != serializerShim) {
- // Wrap shim serializers in an adapter for Spark's unshaded Kryo
- log.debug("Registering {} to serializer shim {} (serializer shim {})",
- tr.getTargetClass(), serializerShim, serializerShim.getClass());
- kryo.register(tr.getTargetClass(), new UnshadedSerializerAdapter<>(serializerShim));
- shimmedClassesFromGryoMapper.add(tr.getTargetClass());
- } else if (null != functionOfShadedKryo) {
- // As with shaded serializers, there's supposed to be a check in GryoMapper that prevents this from happening
- log.error("GryoMapper's default serialization registration for {} is a Function<{},{}>. " +
- "This is probably a bug in TinkerPop (this is not a valid default registration). " +
- "I am configuring Spark to use Kryo's default serializer instead of this function, " +
- "but this may cause serialization failures at runtime.",
- tr.getTargetClass(),
- org.apache.tinkerpop.shaded.kryo.Kryo.class.getCanonicalName(),
- org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
- kryo.register(tr.getTargetClass());
- } else {
- // Register all other classes with the default behavior (FieldSerializer)
- log.debug("Registering {} with default serializer", tr.getTargetClass());
- kryo.register(tr.getTargetClass());
- }
- }
-
- // StarGraph's shim serializer is especially important on Spark for efficiency reasons,
- // so log a warning if we failed to register it somehow
- if (!shimmedClassesFromGryoMapper.contains(StarGraph.class)) {
- log.warn("No SerializerShim found for StarGraph");
- }
- }
-
- private LinkedHashMap<Class<?>, Serializer<?>> getExtraRegistrations() {
-
- /* The map returned by this method MUST have a fixed iteration order!
- *
- * The order itself is irrelevant, so long as it is completely stable at runtime.
- *
- * LinkedHashMap satisfies this requirement (its contract specifies
- * iteration in key-insertion-order).
- */
-
- LinkedHashMap<Class<?>, Serializer<?>> m = new LinkedHashMap<>();
- // The following entries were copied from GryoSerializer's constructor
- // This could be turned into a static collection on GryoSerializer to avoid
- // duplication, but it would be a bit cumbersome to do so without disturbing
- // the ordering of the existing entries in that constructor, since not all
- // of the entries are for TinkerPop (and the ordering is significant).
- m.put(MessagePayload.class, null);
- m.put(ViewIncomingPayload.class, null);
- m.put(ViewOutgoingPayload.class, null);
- m.put(ViewPayload.class, null);
- m.put(VertexWritable.class, new UnshadedSerializerAdapter<>(new VertexWritableSerializer()));
- m.put(ObjectWritable.class, new UnshadedSerializerAdapter<>(new ObjectWritableSerializer<>()));
-
- return m;
- }
-
- private boolean checkForAndApplySerializerOverride(Map<Class<?>, Serializer<?>> serializerOverrides,
- Kryo kryo, Class<?> targetClass) {
- if (serializerOverrides.containsKey(targetClass)) {
- Serializer<?> ser = serializerOverrides.get(targetClass);
- if (null == ser) {
- // null means use Kryo's default serializer
- log.debug("Registering {} with default serializer per overrides", targetClass);
- kryo.register(targetClass);
- } else {
- // nonnull means use that serializer
- log.debug("Registering {} with serializer {} per overrides", targetClass, ser);
- kryo.register(targetClass, ser);
- }
- return true;
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/88ac3047/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
index a524a97..95ebb0d 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
@@ -98,7 +98,7 @@ public class UnshadedKryoShimService implements KryoShimService {
@Override
public int getPriority() {
- return 50;
+ return -50;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/88ac3047/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index cf76027..85552ce 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.spark.process.computer;
+import org.apache.spark.serializer.KryoSerializer;
import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.groovy.util.SugarTestHelper;
@@ -37,6 +38,7 @@ import org.apache.tinkerpop.gremlin.spark.structure.Spark;
import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorageCheck;
import org.apache.tinkerpop.gremlin.spark.structure.io.ToyGraphInputRDD;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -79,9 +81,14 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
config.put(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
- /// spark configuration
+
config.put("spark.master", "local[4]");
- config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
+ if (false) {
+ config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
+ } else {
+ config.put("spark.serializer", KryoSerializer.class.getCanonicalName());
+ config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName());
+ }
config.put("spark.kryo.registrationRequired", true);
return config;
}
[07/18] incubator-tinkerpop git commit: Fix MonthDay serializer
Posted by ok...@apache.org.
Fix MonthDay serializer
I carelessly broke the read leg of this serializer when porting it to
the shim. Ireplaced actual deserialization logic with autogenerated
"return null" when inserting a method template with the shim's
slightly different method signature. Trivial fix though.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ebd2f857
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ebd2f857
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ebd2f857
Branch: refs/heads/master
Commit: ebd2f8576acdb472fa8696b58db128eac191f0ef
Parents: 9321a3e
Author: Dan LaRocque <da...@hopcount.org>
Authored: Mon Jun 6 04:35:34 2016 -0400
Committer: Dan LaRocque <da...@hopcount.org>
Committed: Mon Jun 6 04:35:34 2016 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ebd2f857/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
index 8b14345..24df032 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
@@ -139,7 +139,7 @@ final class JavaTimeSerializers {
@Override
public <I extends InputShim> MonthDay read(KryoShim<I, ?> kryo, I input, Class<MonthDay> clazz) {
- return null;
+ return MonthDay.of(input.readInt(), input.readInt());
}
}
[17/18] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-1321'
Posted by ok...@apache.org.
Merge branch 'TINKERPOP-1321'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4bb9f36a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4bb9f36a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4bb9f36a
Branch: refs/heads/master
Commit: 4bb9f36ae11304fc469cd2fb4184bd31bd3a4d27
Parents: 406956d 45abdcc
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Jun 7 06:46:29 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Jun 7 06:46:29 2016 -0600
----------------------------------------------------------------------
.../process/computer/GiraphWorkerContext.java | 3 +-
.../gremlin/structure/io/gryo/GryoMapper.java | 443 ++++++++++++-------
.../gremlin/structure/io/gryo/GryoPool.java | 1 +
.../structure/io/gryo/GryoSerializers.java | 40 +-
.../structure/io/gryo/JavaTimeSerializers.java | 125 ++----
.../structure/io/gryo/PairSerializer.java | 11 +-
.../structure/io/gryo/TypeRegistration.java | 82 ++++
.../structure/io/gryo/URISerializer.java | 23 +-
.../structure/io/gryo/UUIDSerializer.java | 24 +-
.../structure/io/gryo/kryoshim/InputShim.java | 38 ++
.../structure/io/gryo/kryoshim/KryoShim.java | 41 ++
.../io/gryo/kryoshim/KryoShimService.java | 99 +++++
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 215 +++++++++
.../structure/io/gryo/kryoshim/OutputShim.java | 40 ++
.../io/gryo/kryoshim/SerializerShim.java | 36 ++
.../io/gryo/kryoshim/package-info.java | 55 +++
.../kryoshim/shaded/ShadedInputAdapter.java | 66 +++
.../gryo/kryoshim/shaded/ShadedKryoAdapter.java | 67 +++
.../kryoshim/shaded/ShadedOutputAdapter.java | 71 +++
.../shaded/ShadedSerializerAdapter.java | 54 +++
.../io/gryo/kryoshim/shaded/package-info.java | 25 ++
.../util/star/StarGraphGryoSerializer.java | 130 +-----
.../util/star/StarGraphSerializer.java | 150 +++++++
.../hadoop/process/computer/HadoopCombine.java | 3 +-
.../hadoop/process/computer/HadoopMap.java | 3 +-
.../hadoop/process/computer/HadoopReduce.java | 3 +-
.../structure/io/HadoopPoolShimService.java | 75 ++++
.../hadoop/structure/io/HadoopPools.java | 1 +
.../structure/io/HadoopPoolsConfigurable.java | 4 +-
.../hadoop/structure/io/ObjectWritable.java | 25 +-
.../hadoop/structure/io/VertexWritable.java | 39 +-
.../structure/io/gryo/GryoRecordReader.java | 3 +-
.../structure/io/gryo/GryoRecordWriter.java | 4 +-
...n.structure.io.gryo.kryoshim.KryoShimService | 1 +
.../gremlin/hadoop/HadoopGraphProvider.java | 4 +
.../spark/process/computer/SparkExecutor.java | 14 +-
.../structure/io/gryo/GryoRegistrator.java | 238 ++++++++++
.../spark/structure/io/gryo/GryoSerializer.java | 2 +-
.../io/gryo/IoRegistryAwareKryoSerializer.java | 116 +++++
.../io/gryo/ObjectWritableSerializer.java | 12 +-
.../io/gryo/VertexWritableSerializer.java | 12 +-
.../kryoshim/unshaded/UnshadedInputAdapter.java | 78 ++++
.../kryoshim/unshaded/UnshadedKryoAdapter.java | 74 ++++
.../unshaded/UnshadedKryoShimService.java | 154 +++++++
.../unshaded/UnshadedOutputAdapter.java | 83 ++++
.../unshaded/UnshadedSerializerAdapter.java | 57 +++
...n.structure.io.gryo.kryoshim.KryoShimService | 1 +
.../gremlin/spark/AbstractSparkTest.java | 4 +
...tratorGraphComputerProcessIntegrateTest.java | 33 ++
...SparkHadoopGraphGryoRegistratorProvider.java | 56 +++
.../computer/SparkHadoopGraphProvider.java | 16 +-
.../spark/structure/io/ToyGraphInputRDD.java | 3 +-
52 files changed, 2480 insertions(+), 477 deletions(-)
----------------------------------------------------------------------
[11/18] incubator-tinkerpop git commit: VertexProgramHelper now uses
Gryo to serialize objects if the standard Java Serializer fails. Moving
forward (3.3.x), this should all be accomplished by KryoShimServiceLoader.
Posted by ok...@apache.org.
VertexProgramHelper now uses Gryo to serialize objects if the standard Java Serializer fails. Moving forward (3.3.x), this should all be accomplished by KryoShimServiceLoader.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/797364cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/797364cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/797364cb
Branch: refs/heads/master
Commit: 797364cb4b5c4d0bd18d59a3a7c6cdb5603e136c
Parents: e700363
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Jun 6 16:37:38 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Jun 6 16:37:38 2016 -0600
----------------------------------------------------------------------
.../computer/util/VertexProgramHelper.java | 33 +++++++++++++++-----
.../gremlin/structure/io/gryo/GryoMapper.java | 17 +++++-----
.../gremlin/hadoop/HadoopGraphProvider.java | 4 +++
.../structure/io/gryo/GryoRegistrator.java | 24 +++-----------
4 files changed, 45 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/797364cb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
index bc67866..2b3a0b2 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
@@ -25,8 +25,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.util.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
@@ -37,6 +42,8 @@ import java.util.Set;
*/
public final class VertexProgramHelper {
+ private static final GryoPool GRYO_POOL = GryoPool.build().create();
+
private VertexProgramHelper() {
}
@@ -67,21 +74,33 @@ public final class VertexProgramHelper {
final String byteString = Arrays.toString(Serializer.serializeObject(object));
configuration.setProperty(key, byteString.substring(1, byteString.length() - 1));
} catch (final IOException e) {
- throw new IllegalArgumentException(e.getMessage(), e);
+ try {
+ final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ GRYO_POOL.doWithWriter(kryo -> kryo.writeObject(outputStream, object));
+ String byteString = Arrays.toString(outputStream.toByteArray());
+ configuration.setProperty(key, byteString.substring(1, byteString.length() - 1));
+ } catch (final Exception e1) {
+ throw new IllegalArgumentException(e1.getMessage(), e1);
+ }
}
}
public static <T> T deserialize(final Configuration configuration, final String key) {
+ final String[] stringBytes = configuration.getString(key).split(",");
+ byte[] bytes = new byte[stringBytes.length];
+ for (int i = 0; i < stringBytes.length; i++) {
+ bytes[i] = Byte.valueOf(stringBytes[i].trim());
+ }
try {
- final String[] stringBytes = configuration.getString(key).split(",");
- byte[] bytes = new byte[stringBytes.length];
- for (int i = 0; i < stringBytes.length; i++) {
- bytes[i] = Byte.valueOf(stringBytes[i].trim());
- }
return (T) Serializer.deserializeObject(bytes);
} catch (final IOException | ClassNotFoundException e) {
- throw new IllegalArgumentException(e.getMessage(), e);
+ try {
+ return (T) GRYO_POOL.readWithKryo(kryo -> kryo.readClassAndObject(new Input(new ByteArrayInputStream(bytes))));
+ } catch (final Exception e1) {
+ throw new IllegalArgumentException(e1.getMessage(), e1);
+ }
}
+
}
public static <S, E> Traversal.Admin<S, E> reverse(final Traversal.Admin<S, E> traversal) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/797364cb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 41ca44d..7bf9b7d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -48,6 +48,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSe
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalMetrics;
import org.apache.tinkerpop.gremlin.process.traversal.util.ImmutableMetrics;
import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
+import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalExplanation;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
@@ -330,6 +331,8 @@ public final class GryoMapper implements Mapper<Kryo> {
add(GryoTypeReg.of(AtomicLong.class, 79));
add(GryoTypeReg.of(Pair.class, 88, new PairSerializer()));
add(GryoTypeReg.of(TraversalExplanation.class, 106, new JavaSerializer()));
+ add(GryoTypeReg.of(GraphFilter.class, 120, new JavaSerializer())); // ***LAST ID***
+ //add(GryoTypeReg.of(PureTraversal.class, 121, new JavaSerializer()));
add(GryoTypeReg.of(Duration.class, 93, new JavaTimeSerializers.DurationSerializer()));
add(GryoTypeReg.of(Instant.class, 94, new JavaTimeSerializers.InstantSerializer()));
@@ -355,7 +358,7 @@ public final class GryoMapper implements Mapper<Kryo> {
add(GryoTypeReg.of(GroupStepV3d0.GroupBiOperatorV3d0.class, 113));
add(GryoTypeReg.of(RangeGlobalStep.RangeBiOperator.class, 114));
add(GryoTypeReg.of(OrderGlobalStep.OrderBiOperator.class, 118, new JavaSerializer())); // because they contain traversals
- add(GryoTypeReg.of(ProfileStep.ProfileBiOperator.class, 119)); // ***LAST ID***
+ add(GryoTypeReg.of(ProfileStep.ProfileBiOperator.class, 119));
}};
private final List<IoRegistry> registries = new ArrayList<>();
@@ -374,12 +377,12 @@ public final class GryoMapper implements Mapper<Kryo> {
// For justification of these default registration rules, see TinkerPopKryoRegistrator
for (TypeRegistration<?> tr : typeRegistrations) {
if (tr.hasSerializer() /* no serializer is acceptable */ &&
- null == tr.getSerializerShim() /* a shim serializer is acceptable */ &&
- !(tr.getShadedSerializer() instanceof JavaSerializer) /* shaded JavaSerializer is acceptable */) {
+ null == tr.getSerializerShim() /* a shim serializer is acceptable */ &&
+ !(tr.getShadedSerializer() instanceof JavaSerializer) /* shaded JavaSerializer is acceptable */) {
// everything else is invalid
String msg = String.format("The default GryoMapper type registration %s is invalid. " +
- "It must supply either an implementation of %s or %s, but supplies neither. " +
- "This is probably a bug in GryoMapper's default serialization class registrations.", tr,
+ "It must supply either an implementation of %s or %s, but supplies neither. " +
+ "This is probably a bug in GryoMapper's default serialization class registrations.", tr,
SerializerShim.class.getCanonicalName(), JavaSerializer.class.getCanonicalName());
throw new IllegalStateException(msg);
}
@@ -553,8 +556,8 @@ public final class GryoMapper implements Mapper<Kryo> {
if (1 < serializerCount) {
String msg = String.format(
"GryoTypeReg accepts at most one kind of serializer, but multiple " +
- "serializers were supplied for class %s (id %s). " +
- "Shaded serializer: %s. Shim serializer: %s. Shaded serializer function: %s.",
+ "serializers were supplied for class %s (id %s). " +
+ "Shaded serializer: %s. Shim serializer: %s. Shaded serializer function: %s.",
this.clazz.getCanonicalName(), id,
this.shadedSerializer, this.serializerShim, this.functionOfShadedKryo);
throw new IllegalArgumentException(msg);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/797364cb/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index da00463..57157db 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -28,6 +28,7 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
@@ -45,6 +46,8 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
+import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader.SHIM_CLASS_SYSTEM_PROPERTY;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
* @author Stephen Mallette (http://stephen.genoprime.com)
@@ -109,6 +112,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, HadoopPoolShimService.class.getCanonicalName());
this.graphSONInput = RANDOM.nextBoolean();
return new HashMap<String, Object>() {{
put(Graph.GRAPH, HadoopGraph.class.getName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/797364cb/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
index 9563408..68112d7 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
@@ -22,8 +22,6 @@ import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
import org.apache.spark.serializer.KryoRegistrator;
-import org.apache.spark.util.SerializableConfiguration;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
@@ -31,37 +29,27 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
-import org.apache.tinkerpop.gremlin.process.traversal.Path;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.ImmutablePath;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.MutablePath;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_LP_O_P_S_SE_SL_Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_LP_O_S_SE_SL_Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_O_S_SE_SL_Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_O_Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.LP_O_OB_P_S_SE_SL_Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.LP_O_OB_S_SE_SL_Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.O_OB_S_SE_SL_Traverser;
-import org.apache.tinkerpop.gremlin.process.traversal.traverser.O_Traverser;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedSerializerAdapter;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Property;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoSerializers;
import org.apache.tinkerpop.gremlin.structure.io.gryo.TypeRegistration;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.collection.mutable.WrappedArray;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
/**
* A spark.kryo.registrator implementation that installs TinkerPop types.
@@ -205,8 +193,6 @@ public class GryoRegistrator implements KryoRegistrator {
m.put(VertexWritable.class, new UnshadedSerializerAdapter<>(new VertexWritableSerializer()));
m.put(ObjectWritable.class, new UnshadedSerializerAdapter<>(new ObjectWritableSerializer<>()));
//
- m.put(HadoopConfiguration.class, null);
- //
m.put(HadoopVertex.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexSerializer()));
m.put(HadoopVertexProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexPropertySerializer()));
m.put(HadoopProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.PropertySerializer()));
[18/18] incubator-tinkerpop git commit: removed GraphFilter from
GryoMapper. It was added as another way of solving a problem,
but that solution wasn't used so removing it. Also,
realized a cleaner way of doing the KryoShim tests in Spark -- minor config
Posted by ok...@apache.org.
removed GraphFilter from GryoMapper. It was added as another way of solving a problem, but that solution wasn't used so removing it. Also, realized a cleaner way of doing the KryoShim tests in Spark -- minor configuration changes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2c7b1f92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2c7b1f92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2c7b1f92
Branch: refs/heads/master
Commit: 2c7b1f92ba0a591ef3f25f75447b37a0edb60fd9
Parents: 4bb9f36
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Jun 7 07:14:40 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Jun 7 07:14:40 2016 -0600
----------------------------------------------------------------------
.../gremlin/structure/io/gryo/GryoMapper.java | 4 +---
.../gremlin/spark/AbstractSparkTest.java | 1 -
.../SparkHadoopGraphGryoRegistratorProvider.java | 18 +++++++-----------
.../computer/SparkHadoopGraphProvider.java | 1 -
4 files changed, 8 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2c7b1f92/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 577c1ce..5b4a3a8 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -330,8 +330,6 @@ public final class GryoMapper implements Mapper<Kryo> {
add(GryoTypeReg.of(AtomicLong.class, 79));
add(GryoTypeReg.of(Pair.class, 88, new PairSerializer()));
add(GryoTypeReg.of(TraversalExplanation.class, 106, new JavaSerializer()));
- add(GryoTypeReg.of(GraphFilter.class, 120, new JavaSerializer())); // ***LAST ID***
- //add(GryoTypeReg.of(PureTraversal.class, 121, new JavaSerializer()));
add(GryoTypeReg.of(Duration.class, 93, new JavaTimeSerializers.DurationSerializer()));
add(GryoTypeReg.of(Instant.class, 94, new JavaTimeSerializers.InstantSerializer()));
@@ -357,7 +355,7 @@ public final class GryoMapper implements Mapper<Kryo> {
add(GryoTypeReg.of(GroupStepV3d0.GroupBiOperatorV3d0.class, 113));
add(GryoTypeReg.of(RangeGlobalStep.RangeBiOperator.class, 114));
add(GryoTypeReg.of(OrderGlobalStep.OrderBiOperator.class, 118, new JavaSerializer())); // because they contain traversals
- add(GryoTypeReg.of(ProfileStep.ProfileBiOperator.class, 119));
+ add(GryoTypeReg.of(ProfileStep.ProfileBiOperator.class, 119)); // ***LAST ID***
}};
private final List<IoRegistry> registries = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2c7b1f92/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
index 4d584ff..473976b 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/AbstractSparkTest.java
@@ -46,7 +46,6 @@ public abstract class AbstractSparkTest {
@After
@Before
public void setupTest() {
- System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY);
SparkConf sparkConfiguration = new SparkConf();
sparkConfiguration.setAppName(this.getClass().getCanonicalName() + "-setupTest");
sparkConfiguration.set("spark.master", "local[4]");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2c7b1f92/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
index df36a13..c1832cc 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphGryoRegistratorProvider.java
@@ -36,21 +36,17 @@ import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimSe
*/
public final class SparkHadoopGraphGryoRegistratorProvider extends SparkHadoopGraphProvider {
- private static boolean firstTest = true;
-
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ Spark.close();
final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false); // ensure the context doesn't stay open for the GryoSerializer tests
+ // ensure the context doesn't stay open for the GryoSerializer tests to follow
+ // this is primarily to ensure that the KryoShimService loaded specifically in these tests don't leak to the other tests
+ config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, false);
config.put("spark.serializer", KryoSerializer.class.getCanonicalName());
config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName());
- //
- if (firstTest) {
- firstTest = false;
- Spark.close();
- System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, UnshadedKryoShimService.class.getCanonicalName());
- KryoShimServiceLoader.load(true);
- }
- //
+ System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, UnshadedKryoShimService.class.getCanonicalName());
+ KryoShimServiceLoader.load(true);
+ System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY);
return config;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/2c7b1f92/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 1fc29cf..06119fc 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -88,7 +88,6 @@ public class SparkHadoopGraphProvider extends HadoopGraphProvider {
SugarTestHelper.clearRegistry(this);
}
- System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY);
config.put(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName());
config.put("spark.master", "local[4]");
config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
[13/18] incubator-tinkerpop git commit: remove uneeded import.
Posted by ok...@apache.org.
remove uneeded import.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/359f04bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/359f04bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/359f04bb
Branch: refs/heads/master
Commit: 359f04bbc314d69d5b03f0f241c69f629dbac13a
Parents: 90e3159
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Jun 6 16:57:16 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Jun 6 16:57:16 2016 -0600
----------------------------------------------------------------------
.../gremlin/hadoop/structure/io/HadoopPoolShimService.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/359f04bb/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
index df72b71..12b5d07 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.hadoop.structure.io;
import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.io.Input;
import org.apache.tinkerpop.shaded.kryo.io.Output;
[04/18] incubator-tinkerpop git commit: Kryo shim refactoring and
documentation
Posted by ok...@apache.org.
Kryo shim refactoring and documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/218d7909
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/218d7909
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/218d7909
Branch: refs/heads/master
Commit: 218d7909b9c35a4488102f53c444c92be737972c
Parents: ef52869
Author: Dan LaRocque <da...@hopcount.org>
Authored: Fri Jun 3 07:41:50 2016 -0400
Committer: Dan LaRocque <da...@hopcount.org>
Committed: Fri Jun 3 07:41:50 2016 -0400
----------------------------------------------------------------------
.../gremlin/structure/io/gryo/GryoMapper.java | 29 +--
.../structure/io/gryo/TypeRegistration.java | 2 +-
.../structure/io/gryo/URISerializer.java | 8 +-
.../structure/io/gryo/UUIDSerializer.java | 12 +-
.../structure/io/gryo/kryoshim/InputShim.java | 38 ++++
.../structure/io/gryo/kryoshim/KryoShim.java | 41 ++++
.../io/gryo/kryoshim/KryoShimService.java | 83 ++++++++
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 199 +++++++++++++++++++
.../structure/io/gryo/kryoshim/OutputShim.java | 40 ++++
.../io/gryo/kryoshim/SerializerShim.java | 36 ++++
.../io/gryo/kryoshim/package-info.java | 55 +++++
.../kryoshim/shaded/ShadedInputAdapter.java | 66 ++++++
.../gryo/kryoshim/shaded/ShadedKryoAdapter.java | 67 +++++++
.../kryoshim/shaded/ShadedOutputAdapter.java | 72 +++++++
.../shaded/ShadedSerializerAdapter.java | 54 +++++
.../io/gryo/kryoshim/shaded/package-info.java | 25 +++
.../structure/io/kryoshim/InputShim.java | 37 ----
.../gremlin/structure/io/kryoshim/KryoShim.java | 40 ----
.../structure/io/kryoshim/KryoShimService.java | 83 --------
.../io/kryoshim/KryoShimServiceLoader.java | 137 -------------
.../structure/io/kryoshim/OutputShim.java | 41 ----
.../structure/io/kryoshim/SerializerShim.java | 35 ----
.../structure/io/kryoshim/package-info.java | 54 -----
.../io/kryoshim/shaded/ShadedInputAdapter.java | 66 ------
.../io/kryoshim/shaded/ShadedKryoAdapter.java | 67 -------
.../io/kryoshim/shaded/ShadedOutputAdapter.java | 72 -------
.../shaded/ShadedSerializerAdapter.java | 54 -----
.../io/kryoshim/shaded/package-info.java | 25 ---
.../util/star/StarGraphGryoSerializer.java | 9 +-
.../util/star/StarGraphSerializer.java | 18 +-
.../structure/io/HadoopPoolShimService.java | 2 +-
.../hadoop/structure/io/ObjectWritable.java | 4 +-
.../hadoop/structure/io/VertexWritable.java | 3 +-
...n.structure.io.gryo.kryoshim.KryoShimService | 1 +
...remlin.structure.io.kryoshim.KryoShimService | 1 -
.../structure/io/TinkerPopKryoRegistrator.java | 121 +++++++++++
.../io/gryo/ObjectWritableSerializer.java | 12 +-
.../io/gryo/VertexWritableSerializer.java | 12 +-
.../kryoshim/unshaded/UnshadedInputAdapter.java | 78 ++++++++
.../kryoshim/unshaded/UnshadedKryoAdapter.java | 74 +++++++
.../unshaded/UnshadedKryoShimService.java | 115 +++++++++++
.../unshaded/UnshadedOutputAdapter.java | 83 ++++++++
.../unshaded/UnshadedSerializerAdapter.java | 57 ++++++
43 files changed, 1368 insertions(+), 760 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 5bc71da..851b03c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -57,8 +57,8 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.Mapper;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded.ShadedSerializerAdapter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.shaded.ShadedSerializerAdapter;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPath;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty;
@@ -70,8 +70,6 @@ import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceProperty;
import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex;
import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexProperty;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphGryoSerializer;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphSerializer;
import org.apache.tinkerpop.shaded.kryo.ClassResolver;
import org.apache.tinkerpop.shaded.kryo.Kryo;
@@ -115,7 +113,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeMap;
@@ -406,7 +403,7 @@ public final class GryoMapper implements Mapper<Kryo> {
public Builder addCustom(final Class... custom) {
if (custom != null && custom.length > 0) {
for (Class c : custom) {
- addOrOverrideRegistration(GryoTypeReg.of(c, currentSerializationId.getAndIncrement()));
+ addOrOverrideRegistration(c, id -> GryoTypeReg.of(c, id));
}
}
return this;
@@ -417,7 +414,7 @@ public final class GryoMapper implements Mapper<Kryo> {
* a class that is already registered will override that registration.
*/
public Builder addCustom(final Class clazz, final Serializer serializer) {
- addOrOverrideRegistration(GryoTypeReg.of(clazz, currentSerializationId.getAndIncrement(), serializer));
+ addOrOverrideRegistration(clazz, id -> GryoTypeReg.of(clazz, id, serializer));
return this;
}
@@ -425,7 +422,7 @@ public final class GryoMapper implements Mapper<Kryo> {
* Register custom class to serialize with a custom serialization shim.
*/
public Builder addCustom(final Class clazz, final SerializerShim serializer) {
- addOrOverrideRegistration(GryoTypeReg.of(clazz, currentSerializationId.getAndIncrement(), serializer));
+ addOrOverrideRegistration(clazz, id -> GryoTypeReg.of(clazz, id, serializer));
return this;
}
@@ -434,7 +431,7 @@ public final class GryoMapper implements Mapper<Kryo> {
* that calling this method for a class that is already registered will override that registration.
*/
public Builder addCustom(final Class clazz, final Function<Kryo, Serializer> functionOfKryo) {
- addOrOverrideRegistration(GryoTypeReg.of(clazz, currentSerializationId.getAndIncrement(), functionOfKryo));
+ addOrOverrideRegistration(clazz, id -> GryoTypeReg.of(clazz, id, functionOfKryo));
return this;
}
@@ -490,16 +487,24 @@ public final class GryoMapper implements Mapper<Kryo> {
return new GryoMapper(this);
}
- private <T> void addOrOverrideRegistration(TypeRegistration<T> newRegistration) {
+ private <T> void addOrOverrideRegistration(Class<?> clazz, Function<Integer, TypeRegistration<T>> newRegistrationBuilder) {
Iterator<TypeRegistration<?>> iter = typeRegistrations.iterator();
+ Integer registrationId = null;
while (iter.hasNext()) {
TypeRegistration<?> existingRegistration = iter.next();
- if (existingRegistration.getTargetClass().equals(newRegistration.getTargetClass())) {
+ if (existingRegistration.getTargetClass().equals(clazz)) {
+ // when overridding a registration, use the old id
+ registrationId = existingRegistration.getId();
+ // remove the old registration (we install its override below)
iter.remove();
break;
}
}
- typeRegistrations.add(newRegistration);
+ if (null == registrationId) {
+ // when not overridding a registration, get an id from the counter
+ registrationId = currentSerializationId.getAndIncrement();
+ }
+ typeRegistrations.add(newRegistrationBuilder.apply(registrationId));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
index 5ca3f31..ef105ce 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
@@ -18,7 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Serializer;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
index de08061..a65bcea 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
@@ -18,10 +18,10 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import java.net.URI;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
index b86ddc2..d1d59d3 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
@@ -18,14 +18,10 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import java.util.UUID;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java
new file mode 100644
index 0000000..23bec16
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.io.Input}-like abstraction.
+ * See that class for method documentation.
+ */
+public interface InputShim {
+
+ byte readByte();
+
+ byte[] readBytes(int size);
+
+ String readString();
+
+ long readLong();
+
+ int readInt();
+
+ double readDouble();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java
new file mode 100644
index 0000000..a0f3f87
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.Kryo}-like abstraction.
+ * See that class for method documentation.
+ *
+ * @param <I> this interface's complementary InputShim
+ * @param <O> this interface's complementary OutputShim
+ */
+public interface KryoShim<I extends InputShim, O extends OutputShim> {
+
+ <T> T readObject(I input, Class<T> type);
+
+ Object readClassAndObject(I input);
+
+ void writeObject(O output, Object object);
+
+ void writeClassAndObject(O output, Object object);
+
+ <T> T readObjectOrNull(I input, Class<T> type);
+
+ void writeObjectOrNull(O output, Object object, Class type);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
new file mode 100644
index 0000000..959605c
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * This interface exists to decouple HadoopPools from TinkerPop's shaded Kryo.
+ * <p>
+ * VertexWritable and ObjectWritable formerly implemented Serializable by
+ * resorting to statically-pooled shaded Kryo instances maintained by the HadoopPools class.
+ * This is awkward because those shaded Kryo instances require class registration by default.
+ * <p>
+ * Consider what happens with custom property datatypes reachable from the reference graph rooted at an ObjectWritable
+ * or VertexWritable instance. It is not enough for these property classes to merely implement
+ * Serializable, though one think that from skimming ObjectWritable/VertexWritable. Those classes
+ * must also register with TinkerPop's internal, shaded Kryo instances as maintained by HadoopPools,
+ * or else configure those instances to accept unregistered classes.
+ * Otherwise, TinkerPop's shaded Kryo will refuse to serialize those properties (even though
+ * they implement Serializable, and even though the user might think they are only using
+ * Java's standard Serialization mechanism!).
+ * <p>
+ * By hiding the mechanics of serialization behind this interface instead of hardcoding it in
+ * HadoopPools, the user can decide how to implement serialization for ObjectWritable/VertexWritable
+ * (and whatever other classes in TinkerPop decide to implement Serializable but then delegate
+ * all of the implementation details, like ObjectWritable/VertexWritable do now).
+ */
+public interface KryoShimService {
+
+ /**
+ * Deserializes an object from an input stream.
+ *
+ * @param source the stream from which to read an object's serialized form
+ * @return the first deserialized object available from {@code source}
+ */
+ Object readClassAndObject(InputStream source);
+
+ /**
+ * Serializes an object to an output stream. This may flush the output stream.
+ *
+ * @param o the object to serialize
+ * @param sink the stream into which the serialized object is written
+ */
+ void writeClassAndObject(Object o, OutputStream sink);
+
+ /**
+ * Returns this service's relative priority number. Unless explicitly overridden through a
+ * system property ({@link KryoShimServiceLoader#SHIM_CLASS_SYSTEM_PROPERTY}),
+ * the service implementation with the numerically highest priority will be used
+ * and all others ignored. In other words, the highest priority wins (in the absence of a
+ * system property override).
+ * <p>
+ * TinkerPop's current default implementation uses priority value zero.
+ * <p>
+ * Third-party implementations of this interface should (but are not technically required)
+ * to use a priority value with absolute value greater than 100.
+ * <p>
+ * The implementation currently breaks priority ties by lexicographical comparison of
+ * fully-qualified package-and-classname, but this tie-breaking behavior should be
+ * considered undefined and subject to future change. Ties are ignored if the service
+ * is explicitly set through the system property mentioned above.
+ *
+ * @return this implementation's priority value
+ */
+ int getPriority();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
new file mode 100644
index 0000000..9ccf2de
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.ServiceLoader;
+
+/**
+ * Loads the highest-priority or user-selected {@link KryoShimService}.
+ */
+public class KryoShimServiceLoader {
+
+ private static volatile KryoShimService CACHED_SHIM_SERVICE;
+
+ private static final Logger log = LoggerFactory.getLogger(KryoShimServiceLoader.class);
+
+ /**
+ * Set this system property to the fully-qualified name of a {@link KryoShimService}
+ * package-and-classname to force it into service. Setting this property causes the
+ * priority-selection mechanism ({@link KryoShimService#getPriority()}) to be ignored.
+ */
+ public static final String SHIM_CLASS_SYSTEM_PROPERTY = "tinkerpop.kryo.shim";
+
+ /**
+ * Return a reference to the shim service. This method may return a cached shim service
+ * unless {@code forceReload} is true. Calls to this method need not be externally
+ * synchonized.
+ *
+ * @param forceReload if false, this method may use its internal service cache; if true,
+ * this method must ignore cache, and it must invoke {@link ServiceLoader#reload()}
+ * before selecting a new service to return
+ * @return the shim service
+ */
+ public static KryoShimService load(boolean forceReload) {
+
+ if (null != CACHED_SHIM_SERVICE && !forceReload) {
+ return CACHED_SHIM_SERVICE;
+ }
+
+ ArrayList<KryoShimService> services = new ArrayList<>();
+
+ ServiceLoader<KryoShimService> sl = ServiceLoader.load(KryoShimService.class);
+
+ KryoShimService result = null;
+
+ synchronized (KryoShimServiceLoader.class) {
+ if (forceReload) {
+ sl.reload();
+ }
+
+ for (KryoShimService kss : sl) {
+ services.add(kss);
+ }
+ }
+
+ String shimClass = System.getProperty(SHIM_CLASS_SYSTEM_PROPERTY);
+
+ if (null != shimClass) {
+ for (KryoShimService kss : services) {
+ if (kss.getClass().getCanonicalName().equals(shimClass)) {
+ log.info("Set {} provider to {} ({}) from system property {}={}",
+ KryoShimService.class.getSimpleName(), kss, kss.getClass(),
+ SHIM_CLASS_SYSTEM_PROPERTY, shimClass);
+ result = kss;
+ }
+ }
+ } else {
+ Collections.sort(services, KryoShimServiceComparator.INSTANCE);
+
+ for (KryoShimService kss : services) {
+ log.debug("Found Kryo shim service class {} (priority {})", kss.getClass(), kss.getPriority());
+ }
+
+ if (0 != services.size()) {
+ result = services.get(services.size() - 1);
+ }
+ }
+
+
+ if (null == result) {
+ throw new IllegalStateException("Unable to load KryoShimService");
+ }
+
+ log.info("Set {} provider to {} ({}) because its priority value ({}) is the highest available",
+ KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
+
+ return CACHED_SHIM_SERVICE = result;
+ }
+
+ /**
+ * Equivalent to {@link #load(boolean)} with the parameter {@code true}.
+ *
+ * @return the (possibly cached) shim service
+ */
+ public static KryoShimService load() {
+ return load(false);
+ }
+
+ /**
+ * A loose abstraction of {@link org.apache.tinkerpop.shaded.kryo.Kryo#writeClassAndObject(Output, Object)},
+ * where the {@code output} parameter is an internally-created {@link ByteArrayOutputStream}. Returns
+ * the byte array underlying that stream.
+ *
+ * @param o an object for which the instance and class are serialized
+ * @return the serialized form
+ */
+ public static byte[] writeClassAndObjectToBytes(Object o) {
+ KryoShimService shimService = load();
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ shimService.writeClassAndObject(o, baos);
+
+ return baos.toByteArray();
+ }
+
+ /**
+ * A loose abstraction of {@link org.apache.tinkerpop.shaded.kryo.Kryo#readClassAndObject(Input)},
+ * where the {@code input} parameter is {@code source}. Returns the deserialized object.
+ *
+ * @param source an input stream containing data for a serialized object class and instance
+ * @param <T> the type to which the deserialized object is cast as it is returned
+ * @return the deserialized object
+ */
+ public static <T> T readClassAndObject(InputStream source) {
+ KryoShimService shimService = load();
+
+ return (T)shimService.readClassAndObject(source);
+ }
+
+ /**
+ * Selects the service with greatest {@link KryoShimService#getPriority()}
+ * (not absolute value).
+ *
+ * Breaks ties with lexicographical comparison of classnames where the
+ * name that sorts last is considered to have highest priority. Ideally
+ * nothing should rely on that tiebreaking behavior, but it beats random
+ * selection in case a user ever gets into that situation by accident and
+ * tries to figure out what's going on.
+ */
+ private enum KryoShimServiceComparator implements Comparator<KryoShimService> {
+ INSTANCE;
+
+ @Override
+ public int compare(KryoShimService a, KryoShimService b) {
+ int ap = a.getPriority();
+ int bp = b.getPriority();
+
+ if (ap < bp) {
+ return -1;
+ } else if (bp < ap) {
+ return 1;
+ } else {
+ int result = a.getClass().getCanonicalName().compareTo(b.getClass().getCanonicalName());
+
+ if (0 == result) {
+ log.warn("Found two {} implementations with the same canonical classname: {}. " +
+ "This may indicate a problem with the classpath/classloader such as " +
+ "duplicate or conflicting copies of the file " +
+ "META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService.",
+ a.getClass().getCanonicalName());
+ } else {
+ String winner = 0 < result ? a.getClass().getCanonicalName() : b.getClass().getCanonicalName();
+ log.warn("{} implementations {} and {} are tied with priority value {}. " +
+ "Preferring {} to the other because it has a lexicographically greater classname. " +
+ "Consider setting the system property \"{}\" instead of relying on priority tie-breaking.",
+ KryoShimService.class.getSimpleName(), a, b, ap, winner, SHIM_CLASS_SYSTEM_PROPERTY);
+ }
+
+ return result;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java
new file mode 100644
index 0000000..e4ca3d5
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.io.Output}-like abstraction.
+ * See that class for method documentation.
+ */
+public interface OutputShim {
+
+ void writeByte(byte b);
+
+ void writeBytes(byte[] array, int offset, int count);
+
+ void writeString(String s);
+
+ void writeLong(long l);
+
+ void writeInt(int i);
+
+ void writeDouble(double d);
+
+ void flush();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
new file mode 100644
index 0000000..191cdd8
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.Serializer}-like abstraction.
+ * See that class for method documentation.
+ *
+ * @param <T> the class this serializer reads/writes from/to bytes.
+ */
+public interface SerializerShim<T> {
+
+ <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, T starGraph);
+
+ <I extends InputShim> T read(KryoShim<I, ?> kryo, I input, Class<T> clazz);
+
+ default boolean isImmutable() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/package-info.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/package-info.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/package-info.java
new file mode 100644
index 0000000..70675da
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/package-info.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Abstracts a minimal subset of Kryo types and methods.
+ * <p>
+ * Kryo is often shaded. For instance, TinkerPop's Gryo
+ * serializer relies on a shaded Kryo package.
+ * TinkerPop serializers written against a particular shaded
+ * Kryo package (or an unshaded Kryo package) are compatible
+ * only with that package. In contrast, TinkerPop serializers written
+ * against this abstraction can be used with any shaded or
+ * unshaded Kryo package, so long as the signatures and behavior
+ * of the methods in this package remain stable.
+ * <p>
+ * To show how this is useful, consider
+ * {@link org.apache.tinkerpop.gremlin.structure.util.star.StarGraphSerializer}.
+ * This class has logic unique to TinkerPop that performs
+ * efficient and forward-compatible serialization of
+ * {@link org.apache.tinkerpop.gremlin.structure.util.star.StarGraph}
+ * instances. It takes advantage of package-level visibility
+ * and the fact that it shares a package with its target,
+ * so it would be challenging to cleanly and naturally replicate
+ * (i.e. without package spoofing or runtime visibility overrides).
+ * By implementing
+ * {@link org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim}
+ * instead of, say, Gryo's shaded
+ * {@link org.apache.tinkerpop.shaded.kryo.Serializer},
+ * such a serializer can be used with anybody's Kryo package,
+ * regardless of whether
+ * that package is shaded or not. This lets third-parties reuse
+ * TinkerPop's efficient, internals-aware StarGraph serializer on
+ * their own serialization platform (and without altering
+ * TinkerPop's bytecode, let alone its source).
+ * <p>
+ * The number of types and methods in this
+ * package is deliberately small to reduce the likelihood of a
+ * new Kryo release introducing an incompatible change.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java
new file mode 100644
index 0000000..d11c1c4
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+
+public class ShadedInputAdapter implements InputShim {
+
+ private final Input shadedInput;
+
+ public ShadedInputAdapter(Input shadedInput) {
+ this.shadedInput = shadedInput;
+ }
+
+ Input getShadedInput() {
+ return shadedInput;
+ }
+
+ @Override
+ public byte readByte()
+ {
+ return shadedInput.readByte();
+ }
+
+ @Override
+ public byte[] readBytes(int size) {
+ return shadedInput.readBytes(size);
+ }
+
+ @Override
+ public String readString() {
+ return shadedInput.readString();
+ }
+
+ @Override
+ public long readLong() {
+ return shadedInput.readLong();
+ }
+
+ @Override
+ public int readInt() {
+ return shadedInput.readInt();
+ }
+
+ @Override
+ public double readDouble() {
+ return shadedInput.readDouble();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java
new file mode 100644
index 0000000..4283298
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+
+public class ShadedKryoAdapter implements KryoShim<ShadedInputAdapter, ShadedOutputAdapter> {
+
+ private final Kryo shadedKryo;
+
+ public ShadedKryoAdapter(Kryo shadedKryo) {
+ this.shadedKryo = shadedKryo;
+ }
+
+ @Override
+ public <T> T readObject(ShadedInputAdapter input, Class<T> type)
+ {
+ return shadedKryo.readObject(input.getShadedInput(), type);
+ }
+
+ @Override
+ public Object readClassAndObject(ShadedInputAdapter input)
+ {
+ return shadedKryo.readClassAndObject(input.getShadedInput());
+ }
+
+ @Override
+ public void writeObject(ShadedOutputAdapter output, Object object)
+ {
+ shadedKryo.writeObject(output.getShadedOutput(), object);
+ }
+
+ @Override
+ public void writeClassAndObject(ShadedOutputAdapter output, Object object)
+ {
+ shadedKryo.writeClassAndObject(output.getShadedOutput(), object);
+ }
+
+ @Override
+ public <T> T readObjectOrNull(ShadedInputAdapter input, Class<T> type)
+ {
+ return shadedKryo.readObjectOrNull(input.getShadedInput(), type);
+ }
+
+ @Override
+ public void writeObjectOrNull(ShadedOutputAdapter output, Object object, Class type)
+ {
+ shadedKryo.writeObjectOrNull(output.getShadedOutput(), object, type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java
new file mode 100644
index 0000000..7547466
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+
+public class ShadedOutputAdapter implements OutputShim {
+
+ private final Output shadedOutput;
+
+ public ShadedOutputAdapter(Output shadedOutput) {
+ this.shadedOutput = shadedOutput;
+ }
+
+ @Override
+ public void writeByte(byte b)
+ {
+ shadedOutput.writeByte(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] array, int offset, int count) {
+ shadedOutput.writeBytes(array, offset, count);
+ }
+
+ @Override
+ public void writeString(String s) {
+ shadedOutput.writeString(s);
+ }
+
+ @Override
+ public void writeLong(long l) {
+ shadedOutput.writeLong(l);
+ }
+
+ @Override
+ public void writeInt(int i) {
+ shadedOutput.writeInt(i);
+ }
+
+ @Override
+ public void writeDouble(double d) {
+ shadedOutput.writeDouble(d);
+ }
+
+ @Override
+ public void flush() {
+ shadedOutput.flush();
+ }
+
+ Output getShadedOutput()
+ {
+ return shadedOutput;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java
new file mode 100644
index 0000000..4ce27b1
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+
+public class ShadedSerializerAdapter<T> extends Serializer<T> {
+
+ SerializerShim<T> serializer;
+
+ public ShadedSerializerAdapter(SerializerShim<T> serializer) {
+ this.serializer = serializer;
+ setImmutable(this.serializer.isImmutable());
+ }
+
+ @Override
+ public void write(Kryo kryo, Output output, T t) {
+ /* These adapters could be cached pretty efficiently in instance fields if it were guaranteed that this
+ * class was never subject to concurrent use. That's true of Kryo instances, but it is not clear that
+ * it is true of Serializer instances.
+ */
+ ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
+ ShadedOutputAdapter shadedOutputAdapter = new ShadedOutputAdapter(output);
+ serializer.write(shadedKryoAdapter, shadedOutputAdapter, t);
+ }
+
+ @Override
+ public T read(Kryo kryo, Input input, Class<T> aClass) {
+ // Same caching opportunity as in write(...)
+ ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
+ ShadedInputAdapter shadedInputAdapter = new ShadedInputAdapter(input);
+ return serializer.read(shadedKryoAdapter, shadedInputAdapter, aClass);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/package-info.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/package-info.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/package-info.java
new file mode 100644
index 0000000..d91a3f1
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Implementations of the interfaces in
+ * {@link org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim}
+ * using TinkerPop's shaded copy of Kryo.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.shaded;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java
deleted file mode 100644
index 6d00884..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
-
-/**
- * A minimal {@link org.apache.tinkerpop.shaded.kryo.io.Input}-like abstraction.
- */
-public interface InputShim {
-
- byte readByte();
-
- byte[] readBytes(int size);
-
- String readString();
-
- long readLong();
-
- int readInt();
-
- double readDouble();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java
deleted file mode 100644
index e2a95e6..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
-
-/**
- * A minimal {@link org.apache.tinkerpop.shaded.kryo.Kryo}-like abstraction.
- *
- * @param <I> this interface's complementary InputShim
- * @param <O> this interface's complementary OutputShim
- */
-public interface KryoShim<I extends InputShim, O extends OutputShim> {
-
- <T> T readObject(I input, Class<T> type);
-
- Object readClassAndObject(I input);
-
- void writeObject(O output, Object object);
-
- void writeClassAndObject(O output, Object object);
-
- <T> T readObjectOrNull(I input, Class<T> type);
-
- void writeObjectOrNull(O output, Object object, Class type);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java
deleted file mode 100644
index 024d40c..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-
-/**
- * This interface exists to decouple HadoopPools from TinkerPop's shaded Kryo.
- * <p>
- * VertexWritable and ObjectWritable formerly implemented Serializable by
- * resorting to statically-pooled shaded Kryo instances maintained by the HadoopPools class.
- * This is awkward because those shaded Kryo instances require class registration by default.
- * <p>
- * Consider what happens with custom property datatypes reachable from the reference graph rooted at an ObjectWritable
- * or VertexWritable instance. It is not enough for these property classes to merely implement
- * Serializable, though one think that from skimming ObjectWritable/VertexWritable. Those classes
- * must also register with TinkerPop's internal, shaded Kryo instances as maintained by HadoopPools,
- * or else configure those instances to accept unregistered classes.
- * Otherwise, TinkerPop's shaded Kryo will refuse to serialize those properties (even though
- * they implement Serializable, and even though the user might think they are only using
- * Java's standard Serialization mechanism!).
- * <p>
- * By hiding the mechanics of serialization behind this interface instead of hardcoding it in
- * HadoopPools, the user can decide how to implement serialization for ObjectWritable/VertexWritable
- * (and whatever other classes in TinkerPop decide to implement Serializable but then delegate
- * all of the implementation details, like ObjectWritable/VertexWritable do now).
- */
-public interface KryoShimService {
-
- /**
- * Deserializes an object from an input stream.
- *
- * @param source the stream from which to read an object's serialized form
- * @return the first deserialized object available from {@code source}
- */
- Object readClassAndObject(InputStream source);
-
- /**
- * Serializes an object to an output stream. This may flush the output stream.
- *
- * @param o the object to serialize
- * @param sink the stream into which the serialized object is written
- */
- void writeClassAndObject(Object o, OutputStream sink);
-
- /**
- * Returns this service's relative priority number. Unless explicitly overridden through a
- * system property ({@link KryoShimServiceLoader#SHIM_CLASS_SYSTEM_PROPERTY}),
- * the service implementation with the numerically highest priority will be used
- * and all others ignored. In other words, the highest priority wins (in the absence of a
- * system property override).
- * <p>
- * TinkerPop's current default implementation uses priority value zero.
- * <p>
- * Third-party implementations of this interface should (but are not technically required)
- * to use a priority value with absolute value greater than 100.
- * <p>
- * The implementation currently breaks priority ties by lexicographical comparison of
- * fully-qualified package-and-classname, but this tie-breaking behavior should be
- * considered undefined and subject to future change. Ties are ignored if the service
- * is explicitly set through the system property mentioned above.
- *
- * @return this implementation's priority value
- */
- int getPriority();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java
deleted file mode 100644
index 1d5413d..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.ServiceLoader;
-
-public class KryoShimServiceLoader {
-
- private static volatile KryoShimService CACHED_SHIM_SERVICE;
-
- private static final Logger log = LoggerFactory.getLogger(KryoShimServiceLoader.class);
-
- /**
- * Set this system property to the fully-qualified name of a {@link KryoShimService}
- * package-and-classname to force it into service. Setting this property causes the
- * priority-selection mechanism ({@link KryoShimService#getPriority()}) to be ignored.
- */
- public static final String SHIM_CLASS_SYSTEM_PROPERTY = "tinkerpop.kryo.shim";
-
- public static KryoShimService load(boolean forceReload) {
-
- if (null != CACHED_SHIM_SERVICE && !forceReload) {
- return CACHED_SHIM_SERVICE;
- }
-
- ArrayList<KryoShimService> services = new ArrayList<>();
-
- ServiceLoader<KryoShimService> sl = ServiceLoader.load(KryoShimService.class);
-
- KryoShimService result = null;
-
- synchronized (KryoShimServiceLoader.class) {
- if (forceReload) {
- sl.reload();
- }
-
- for (KryoShimService kss : sl) {
- services.add(kss);
- }
- }
-
- String shimClass = System.getProperty(SHIM_CLASS_SYSTEM_PROPERTY);
-
- if (null != shimClass) {
- for (KryoShimService kss : services) {
- if (kss.getClass().getCanonicalName().equals(shimClass)) {
- log.info("Set {} provider to {} ({}) from system property {}={}",
- KryoShimService.class.getSimpleName(), kss, kss.getClass(),
- SHIM_CLASS_SYSTEM_PROPERTY, shimClass);
- result = kss;
- }
- }
- } else {
- Collections.sort(services, KryoShimServiceComparator.INSTANCE);
-
- for (KryoShimService kss : services) {
- log.debug("Found Kryo shim service class {} (priority {})", kss.getClass(), kss.getPriority());
- }
-
- if (0 != services.size()) {
- result = services.get(services.size() - 1);
- }
- }
-
-
- if (null == result) {
- throw new IllegalStateException("Unable to load KryoShimService");
- }
-
- log.info("Set {} provider to {} ({}) because its priority value ({}) is the highest available",
- KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
-
- return CACHED_SHIM_SERVICE = result;
- }
-
- public static KryoShimService load() {
- return load(false);
- }
-
- public static byte[] writeClassAndObjectToBytes(Object o) {
- KryoShimService shimService = load();
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- shimService.writeClassAndObject(o, baos);
-
- return baos.toByteArray();
- }
-
- public static <T> T readClassAndObject(InputStream source) {
- KryoShimService shimService = load();
-
- return (T)shimService.readClassAndObject(source);
- }
-
- private enum KryoShimServiceComparator implements Comparator<KryoShimService> {
- INSTANCE;
-
- @Override
- public int compare(KryoShimService a, KryoShimService b) {
- int ap = a.getPriority();
- int bp = b.getPriority();
-
- if (ap < bp) {
- return -1;
- } else if (bp < ap) {
- return 1;
- } else {
- return a.getClass().getCanonicalName().compareTo(b.getClass().getCanonicalName());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java
deleted file mode 100644
index 4468434..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
-
-import java.io.IOException;
-
-/**
- * A minimal {@link org.apache.tinkerpop.shaded.kryo.io.Output}-like abstraction.
- */
-public interface OutputShim {
-
- void writeByte(byte b);
-
- void writeBytes(byte[] array, int offset, int count);
-
- void writeString(String s);
-
- void writeLong(long l);
-
- void writeInt(int i);
-
- void writeDouble(double d);
-
- void flush();
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java
deleted file mode 100644
index 686350d..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
-
-/**
- * A minimal {@link org.apache.tinkerpop.shaded.kryo.Serializer}-like abstraction.
- *
- * @param <T> the class this serializer reads/writes from/to bytes.
- */
-public interface SerializerShim<T> {
-
- <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, T starGraph);
-
- <I extends InputShim> T read(KryoShim<I, ?> kryo, I input, Class<T> clazz);
-
- default boolean isImmutable() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java
deleted file mode 100644
index 436f117..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * Abstracts a minimal subset of Kryo types and methods.
- * <p>
- * Kryo is often shaded. For instance, TinkerPop's Gryo
- * serializer relies on a shaded Kryo package.
- * TinkerPop serializers written against a particular shaded
- * Kryo package (or the unshaded Kryo package) are compatible
- * only with that package. In contrast, TinkerPop serializers written
- * against this abstraction can be used with any shaded or
- * unshaded Kryo package, so long as the signatures and behavior
- * of the methods in this package remain stable.
- * <p>
- * To show how this is useful, consider
- * {@link org.apache.tinkerpop.gremlin.structure.util.star.StarGraphSerializer}.
- * This class has logic unique to TinkerPop that performs
- * efficient and forward-compatible serialization of
- * {@link org.apache.tinkerpop.gremlin.structure.util.star.StarGraph}
- * instances. It takes advantage of package-level visibility
- * and the fact that it shares a package with its target,
- * so it would be challenging to cleanly and naturally replicate
- * (i.e. without package spoofing or runtime visibility overrides).
- * By implementing
- * {@link org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim}
- * instead of, say, Gryo's shaded
- * {@link org.apache.tinkerpop.shaded.kryo.Serializer},
- * it can be used with anybody's Kryo package, regardless of whether
- * that package is shaded or not. This lets third-parties reuse
- * TinkerPop's efficient, internals-aware StarGraph serializer on
- * their own serialization platform (and without altering
- * TinkerPop's bytecode, let alone its source).
- * <p>
- * The number of types and methods in this
- * package is deliberately small to reduce the likelihood of a
- * new Kryo release introducing an incompatible change.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java
deleted file mode 100644
index 2872326..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
-
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-
-public class ShadedInputAdapter implements InputShim {
-
- private final Input shadedInput;
-
- public ShadedInputAdapter(Input shadedInput) {
- this.shadedInput = shadedInput;
- }
-
- Input getShadedInput() {
- return shadedInput;
- }
-
- @Override
- public byte readByte()
- {
- return shadedInput.readByte();
- }
-
- @Override
- public byte[] readBytes(int size) {
- return shadedInput.readBytes(size);
- }
-
- @Override
- public String readString() {
- return shadedInput.readString();
- }
-
- @Override
- public long readLong() {
- return shadedInput.readLong();
- }
-
- @Override
- public int readInt() {
- return shadedInput.readInt();
- }
-
- @Override
- public double readDouble() {
- return shadedInput.readDouble();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java
deleted file mode 100644
index 0e85f6f..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
-
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-
-public class ShadedKryoAdapter implements KryoShim<ShadedInputAdapter, ShadedOutputAdapter> {
-
- private final Kryo shadedKryo;
-
- public ShadedKryoAdapter(Kryo shadedKryo) {
- this.shadedKryo = shadedKryo;
- }
-
- @Override
- public <T> T readObject(ShadedInputAdapter input, Class<T> type)
- {
- return shadedKryo.readObject(input.getShadedInput(), type);
- }
-
- @Override
- public Object readClassAndObject(ShadedInputAdapter input)
- {
- return shadedKryo.readClassAndObject(input.getShadedInput());
- }
-
- @Override
- public void writeObject(ShadedOutputAdapter output, Object object)
- {
- shadedKryo.writeObject(output.getShadedOutput(), object);
- }
-
- @Override
- public void writeClassAndObject(ShadedOutputAdapter output, Object object)
- {
- shadedKryo.writeClassAndObject(output.getShadedOutput(), object);
- }
-
- @Override
- public <T> T readObjectOrNull(ShadedInputAdapter input, Class<T> type)
- {
- return shadedKryo.readObjectOrNull(input.getShadedInput(), type);
- }
-
- @Override
- public void writeObjectOrNull(ShadedOutputAdapter output, Object object, Class type)
- {
- shadedKryo.writeObjectOrNull(output.getShadedOutput(), object, type);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java
deleted file mode 100644
index 5ca0ecb..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
-
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
-
-public class ShadedOutputAdapter implements OutputShim {
-
- private final Output shadedOutput;
-
- public ShadedOutputAdapter(Output shadedOutput) {
- this.shadedOutput = shadedOutput;
- }
-
- @Override
- public void writeByte(byte b)
- {
- shadedOutput.writeByte(b);
- }
-
- @Override
- public void writeBytes(byte[] array, int offset, int count) {
- shadedOutput.writeBytes(array, offset, count);
- }
-
- @Override
- public void writeString(String s) {
- shadedOutput.writeString(s);
- }
-
- @Override
- public void writeLong(long l) {
- shadedOutput.writeLong(l);
- }
-
- @Override
- public void writeInt(int i) {
- shadedOutput.writeInt(i);
- }
-
- @Override
- public void writeDouble(double d) {
- shadedOutput.writeDouble(d);
- }
-
- @Override
- public void flush() {
- shadedOutput.flush();
- }
-
- Output getShadedOutput()
- {
- return shadedOutput;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java
deleted file mode 100644
index ebfd0fb..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
-
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
-
-public class ShadedSerializerAdapter<T> extends Serializer<T> {
-
- SerializerShim<T> serializer;
-
- public ShadedSerializerAdapter(SerializerShim<T> serializer) {
- this.serializer = serializer;
- setImmutable(this.serializer.isImmutable());
- }
-
- @Override
- public void write(Kryo kryo, Output output, T t) {
- /* These adapters could be cached pretty efficiently in instance fields if it were guaranteed that this
- * class was never subject to concurrent use. That's true of Kryo instances, but it is not clear that
- * it is true of Serializer instances.
- */
- ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
- ShadedOutputAdapter shadedOutputAdapter = new ShadedOutputAdapter(output);
- serializer.write(shadedKryoAdapter, shadedOutputAdapter, t);
- }
-
- @Override
- public T read(Kryo kryo, Input input, Class<T> aClass) {
- // Same caching opportunity as in write(...)
- ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
- ShadedInputAdapter shadedInputAdapter = new ShadedInputAdapter(input);
- return serializer.read(shadedKryoAdapter, shadedInputAdapter, aClass);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java
deleted file mode 100644
index ea70003..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Implementations of the interfaces in
- * {@link org.apache.tinkerpop.gremlin.structure.io.kryoshim}
- * using TinkerPop's shaded copy of Kryo.
- */
-package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/218d7909/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
index 2acd770..b2379ce 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
@@ -23,14 +23,11 @@ import java.util.Map;
import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded.ShadedSerializerAdapter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.shaded.ShadedSerializerAdapter;
/**
- * Kryo serializer for {@link StarGraph}. Implements an internal versioning capability for backward compatibility.
- * The single byte at the front of the serialization stream denotes the version. That version can be used to choose
- * the correct deserialization mechanism. The limitation is that this versioning won't help with backward
- * compatibility for custom serializers from providers. Providers should be encouraged to write their serializers
- * with backward compatibility in mind.
+ * A wrapper for {@link StarGraphSerializer} that makes it compatible with TinkerPop's
+ * shaded Kryo.
*
* @author Marko A. Rodriguez (http://markorodriguez.com)
* @author Stephen Mallette (http://stephen.genoprime.com)
[10/18] incubator-tinkerpop git commit: Able to now test both shim
and non-shim models in Spark. Also go configuration with ProgramTest working.
Posted by ok...@apache.org.
Able to now test both shim and non-shim models in Spark. Also go configuration with ProgramTest working.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/e7003635
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/e7003635
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/e7003635
Branch: refs/heads/master
Commit: e7003635e27c625b3f30492111f20f4fe4e24eb5
Parents: 0cd31bf
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Jun 6 14:52:53 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Jun 6 14:52:53 2016 -0600
----------------------------------------------------------------------
.../structure/io/gryo/GryoSerializers.java | 8 +-
.../structure/io/gryo/GryoRegistrator.java | 90 ++++++++++++++++----
.../computer/SparkHadoopGraphProvider.java | 11 ++-
3 files changed, 88 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e7003635/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
index 16fbe85..2042a4a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
@@ -47,7 +47,7 @@ public final class GryoSerializers {
/**
* Serializes any {@link Edge} implementation encountered to a {@link DetachedEdge}.
*/
- final static class EdgeSerializer implements SerializerShim<Edge> {
+ public final static class EdgeSerializer implements SerializerShim<Edge> {
@Override
public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Edge edge) {
kryo.writeClassAndObject(output, DetachedFactory.detach(edge, true));
@@ -63,7 +63,7 @@ public final class GryoSerializers {
/**
* Serializes any {@link Vertex} implementation encountered to an {@link DetachedVertex}.
*/
- final static class VertexSerializer implements SerializerShim<Vertex> {
+ public final static class VertexSerializer implements SerializerShim<Vertex> {
@Override
public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Vertex vertex) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertex, true));
@@ -78,7 +78,7 @@ public final class GryoSerializers {
/**
* Serializes any {@link Property} implementation encountered to an {@link DetachedProperty}.
*/
- final static class PropertySerializer implements SerializerShim<Property> {
+ public final static class PropertySerializer implements SerializerShim<Property> {
@Override
public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Property property) {
kryo.writeClassAndObject(output, property instanceof VertexProperty ? DetachedFactory.detach((VertexProperty) property, true) : DetachedFactory.detach(property));
@@ -93,7 +93,7 @@ public final class GryoSerializers {
/**
* Serializes any {@link VertexProperty} implementation encountered to an {@link DetachedVertexProperty}.
*/
- final static class VertexPropertySerializer implements SerializerShim<VertexProperty> {
+ public final static class VertexPropertySerializer implements SerializerShim<VertexProperty> {
@Override
public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, VertexProperty vertexProperty) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertexProperty, true));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e7003635/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
index 1ae8c5c..9563408 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
@@ -22,19 +22,44 @@ import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
import org.apache.spark.serializer.KryoRegistrator;
+import org.apache.spark.util.SerializableConfiguration;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
+import org.apache.tinkerpop.gremlin.process.traversal.Path;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.ImmutablePath;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.MutablePath;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_LP_O_P_S_SE_SL_Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_LP_O_S_SE_SL_Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_O_S_SE_SL_Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.B_O_Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.LP_O_OB_P_S_SE_SL_Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.LP_O_OB_S_SE_SL_Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.O_OB_S_SE_SL_Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.traverser.O_Traverser;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedSerializerAdapter;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoSerializers;
import org.apache.tinkerpop.gremlin.structure.io.gryo.TypeRegistration;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import scala.collection.mutable.WrappedArray;
import java.util.*;
@@ -55,11 +80,11 @@ public class GryoRegistrator implements KryoRegistrator {
* Register TinkerPop's classes with the supplied {@link Kryo} instance
* while honoring optional overrides and optional class blacklist ("blackset"?).
*
- * @param kryo the Kryo serializer instance with which to register types
+ * @param kryo the Kryo serializer instance with which to register types
* @param serializerOverrides serializer mappings that override this class's defaults
- * @param blacklist classes which should not be registered at all, even if there is an override entry
- * or if they would be registered by this class by default (does not affect Kryo's
- * built-in registrations, e.g. String.class).
+ * @param blacklist classes which should not be registered at all, even if there is an override entry
+ * or if they would be registered by this class by default (does not affect Kryo's
+ * built-in registrations, e.g. String.class).
*/
public void registerClasses(Kryo kryo, Map<Class<?>, Serializer<?>> serializerOverrides, Set<Class<?>> blacklist) {
// Apply TinkerPop type registrations copied from GyroSerializer's constructor
@@ -111,11 +136,11 @@ public class GryoRegistrator implements KryoRegistrator {
} else {
// There's supposed to be a check in GryoMapper that prevents this from happening
log.error("GryoMapper's default serialization registration for {} is a {}. " +
- "This is probably a bug in TinkerPop (this is not a valid default registration). " +
- "I am configuring Spark to use Kryo's default serializer for this class, " +
- "but this may cause serialization failures at runtime.",
- tr.getTargetClass(),
- org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
+ "This is probably a bug in TinkerPop (this is not a valid default registration). " +
+ "I am configuring Spark to use Kryo's default serializer for this class, " +
+ "but this may cause serialization failures at runtime.",
+ tr.getTargetClass(),
+ org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
kryo.register(tr.getTargetClass());
}
} else if (null != serializerShim) {
@@ -127,12 +152,12 @@ public class GryoRegistrator implements KryoRegistrator {
} else if (null != functionOfShadedKryo) {
// As with shaded serializers, there's supposed to be a check in GryoMapper that prevents this from happening
log.error("GryoMapper's default serialization registration for {} is a Function<{},{}>. " +
- "This is probably a bug in TinkerPop (this is not a valid default registration). " +
- "I am configuring Spark to use Kryo's default serializer instead of this function, " +
- "but this may cause serialization failures at runtime.",
- tr.getTargetClass(),
- org.apache.tinkerpop.shaded.kryo.Kryo.class.getCanonicalName(),
- org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
+ "This is probably a bug in TinkerPop (this is not a valid default registration). " +
+ "I am configuring Spark to use Kryo's default serializer instead of this function, " +
+ "but this may cause serialization failures at runtime.",
+ tr.getTargetClass(),
+ org.apache.tinkerpop.shaded.kryo.Kryo.class.getCanonicalName(),
+ org.apache.tinkerpop.shaded.kryo.Serializer.class.getCanonicalName());
kryo.register(tr.getTargetClass());
} else {
// Register all other classes with the default behavior (FieldSerializer)
@@ -164,13 +189,46 @@ public class GryoRegistrator implements KryoRegistrator {
// duplication, but it would be a bit cumbersome to do so without disturbing
// the ordering of the existing entries in that constructor, since not all
// of the entries are for TinkerPop (and the ordering is significant).
+ if (Boolean.valueOf(System.getProperty("is.testing", "false"))) {
+ try {
+ m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
+ m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
+ } catch (final ClassNotFoundException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ m.put(WrappedArray.ofRef.class, null);
m.put(MessagePayload.class, null);
m.put(ViewIncomingPayload.class, null);
m.put(ViewOutgoingPayload.class, null);
m.put(ViewPayload.class, null);
m.put(VertexWritable.class, new UnshadedSerializerAdapter<>(new VertexWritableSerializer()));
m.put(ObjectWritable.class, new UnshadedSerializerAdapter<>(new ObjectWritableSerializer<>()));
-
+ //
+ m.put(HadoopConfiguration.class, null);
+ //
+ m.put(HadoopVertex.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexSerializer()));
+ m.put(HadoopVertexProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexPropertySerializer()));
+ m.put(HadoopProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.PropertySerializer()));
+ m.put(HadoopEdge.class, new UnshadedSerializerAdapter<>(new GryoSerializers.EdgeSerializer()));
+ //
+ m.put(ComputerGraph.ComputerVertex.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexSerializer()));
+ m.put(ComputerGraph.ComputerVertexProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexPropertySerializer()));
+ m.put(ComputerGraph.ComputerProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.PropertySerializer()));
+ m.put(ComputerGraph.ComputerEdge.class, new UnshadedSerializerAdapter<>(new GryoSerializers.EdgeSerializer()));
+ //
+ m.put(StarGraph.StarEdge.class, new UnshadedSerializerAdapter<>(new GryoSerializers.EdgeSerializer()));
+ m.put(StarGraph.StarVertex.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexSerializer()));
+ m.put(StarGraph.StarProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.PropertySerializer()));
+ m.put(StarGraph.StarVertexProperty.class, new UnshadedSerializerAdapter<>(new GryoSerializers.VertexPropertySerializer()));
+ //
+ m.put(MutablePath.class, new UnshadedSerializerAdapter<>(new GryoSerializers.PathSerializer()));
+ m.put(ImmutablePath.class, new UnshadedSerializerAdapter<>(new GryoSerializers.PathSerializer()));
+ try {
+ m.put(Class.forName(ImmutablePath.class.getCanonicalName() + "$TailPath"), new UnshadedSerializerAdapter<>(new GryoSerializers.PathSerializer()));
+ } catch (final ClassNotFoundException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
return m;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/e7003635/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 85552ce..7737d1e 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -26,6 +26,7 @@ import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPluginCheck;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorageCheck;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService;
import org.apache.tinkerpop.gremlin.process.computer.Computer;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
@@ -40,10 +41,14 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorageCheck;
import org.apache.tinkerpop.gremlin.spark.structure.io.ToyGraphInputRDD;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer;
+import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.kryoshim.unshaded.UnshadedKryoShimService;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import java.util.Map;
+import static org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader.SHIM_CLASS_SYSTEM_PROPERTY;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@@ -83,9 +88,13 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider {
config.put("spark.master", "local[4]");
- if (false) {
+ if (RANDOM.nextBoolean()) {
+ System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, HadoopPoolShimService.class.getCanonicalName());
+ KryoShimServiceLoader.load(true);
config.put("spark.serializer", GryoSerializer.class.getCanonicalName());
} else {
+ System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, UnshadedKryoShimService.class.getCanonicalName());
+ KryoShimServiceLoader.load(true);
config.put("spark.serializer", KryoSerializer.class.getCanonicalName());
config.put("spark.kryo.registrator", GryoRegistrator.class.getCanonicalName());
}
[16/18] incubator-tinkerpop git commit: Use "final" where required to
be consistent with rest of codebase.
Posted by ok...@apache.org.
Use "final" where required to be consistent with rest of codebase.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/45abdcc3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/45abdcc3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/45abdcc3
Branch: refs/heads/master
Commit: 45abdcc35e9a80036ca669cc1d65ad16d09d2461
Parents: 293fe29
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue Jun 7 08:06:30 2016 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue Jun 7 08:06:30 2016 -0400
----------------------------------------------------------------------
.../gremlin/structure/io/gryo/GryoMapper.java | 32 ++++++------
.../structure/io/gryo/GryoSerializers.java | 20 ++++----
.../structure/io/gryo/JavaTimeSerializers.java | 52 ++++++++++----------
.../structure/io/gryo/PairSerializer.java | 4 +-
.../structure/io/gryo/TypeRegistration.java | 2 +-
.../structure/io/gryo/kryoshim/InputShim.java | 12 ++---
.../structure/io/gryo/kryoshim/KryoShim.java | 12 ++---
.../io/gryo/kryoshim/KryoShimService.java | 8 +--
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 30 +++++------
.../structure/io/gryo/kryoshim/OutputShim.java | 14 +++---
.../io/gryo/kryoshim/SerializerShim.java | 6 +--
.../kryoshim/shaded/ShadedInputAdapter.java | 4 +-
.../gryo/kryoshim/shaded/ShadedKryoAdapter.java | 14 +++---
.../kryoshim/shaded/ShadedOutputAdapter.java | 17 +++----
.../shaded/ShadedSerializerAdapter.java | 14 +++---
.../util/star/StarGraphSerializer.java | 2 +-
.../structure/io/HadoopPoolShimService.java | 8 +--
.../hadoop/structure/io/ObjectWritable.java | 4 +-
.../hadoop/structure/io/VertexWritable.java | 4 +-
.../structure/io/gryo/GryoRegistrator.java | 16 +++---
.../io/gryo/IoRegistryAwareKryoSerializer.java | 8 +--
.../io/gryo/ObjectWritableSerializer.java | 4 +-
.../io/gryo/VertexWritableSerializer.java | 4 +-
.../kryoshim/unshaded/UnshadedInputAdapter.java | 4 +-
.../kryoshim/unshaded/UnshadedKryoAdapter.java | 14 +++---
.../unshaded/UnshadedKryoShimService.java | 22 ++++-----
.../unshaded/UnshadedOutputAdapter.java | 14 +++---
.../unshaded/UnshadedSerializerAdapter.java | 6 +--
28 files changed, 175 insertions(+), 176 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 7bf9b7d..577c1ce 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -48,7 +48,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSe
import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalMetrics;
import org.apache.tinkerpop.gremlin.process.traversal.util.ImmutableMetrics;
import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
-import org.apache.tinkerpop.gremlin.process.traversal.util.PureTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalExplanation;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
@@ -380,7 +379,7 @@ public final class GryoMapper implements Mapper<Kryo> {
null == tr.getSerializerShim() /* a shim serializer is acceptable */ &&
!(tr.getShadedSerializer() instanceof JavaSerializer) /* shaded JavaSerializer is acceptable */) {
// everything else is invalid
- String msg = String.format("The default GryoMapper type registration %s is invalid. " +
+ final String msg = String.format("The default GryoMapper type registration %s is invalid. " +
"It must supply either an implementation of %s or %s, but supplies neither. " +
"This is probably a bug in GryoMapper's default serialization class registrations.", tr,
SerializerShim.class.getCanonicalName(), JavaSerializer.class.getCanonicalName());
@@ -505,11 +504,12 @@ public final class GryoMapper implements Mapper<Kryo> {
return new GryoMapper(this);
}
- private <T> void addOrOverrideRegistration(Class<?> clazz, Function<Integer, TypeRegistration<T>> newRegistrationBuilder) {
- Iterator<TypeRegistration<?>> iter = typeRegistrations.iterator();
+ private <T> void addOrOverrideRegistration(final Class<?> clazz,
+ final Function<Integer, TypeRegistration<T>> newRegistrationBuilder) {
+ final Iterator<TypeRegistration<?>> iter = typeRegistrations.iterator();
Integer registrationId = null;
while (iter.hasNext()) {
- TypeRegistration<?> existingRegistration = iter.next();
+ final TypeRegistration<?> existingRegistration = iter.next();
if (existingRegistration.getTargetClass().equals(clazz)) {
// when overridding a registration, use the old id
registrationId = existingRegistration.getId();
@@ -534,11 +534,11 @@ public final class GryoMapper implements Mapper<Kryo> {
private final Function<Kryo, Serializer> functionOfShadedKryo;
private final int id;
- private GryoTypeReg(Class<T> clazz,
- Serializer<T> shadedSerializer,
- SerializerShim<T> serializerShim,
- Function<Kryo, Serializer> functionOfShadedKryo,
- int id) {
+ private GryoTypeReg(final Class<T> clazz,
+ final Serializer<T> shadedSerializer,
+ final SerializerShim<T> serializerShim,
+ final Function<Kryo, Serializer> functionOfShadedKryo,
+ final int id) {
this.clazz = clazz;
this.shadedSerializer = shadedSerializer;
this.serializerShim = serializerShim;
@@ -554,7 +554,7 @@ public final class GryoMapper implements Mapper<Kryo> {
serializerCount++;
if (1 < serializerCount) {
- String msg = String.format(
+ final String msg = String.format(
"GryoTypeReg accepts at most one kind of serializer, but multiple " +
"serializers were supplied for class %s (id %s). " +
"Shaded serializer: %s. Shim serializer: %s. Shaded serializer function: %s.",
@@ -564,19 +564,19 @@ public final class GryoMapper implements Mapper<Kryo> {
}
}
- private static <T> GryoTypeReg<T> of(Class<T> clazz, int id) {
+ private static <T> GryoTypeReg<T> of(final Class<T> clazz, final int id) {
return new GryoTypeReg<>(clazz, null, null, null, id);
}
- private static <T> GryoTypeReg<T> of(Class<T> clazz, int id, Serializer<T> shadedSerializer) {
+ private static <T> GryoTypeReg<T> of(final Class<T> clazz, final int id, final Serializer<T> shadedSerializer) {
return new GryoTypeReg<>(clazz, shadedSerializer, null, null, id);
}
- private static <T> GryoTypeReg<T> of(Class<T> clazz, int id, SerializerShim<T> serializerShim) {
+ private static <T> GryoTypeReg<T> of(final Class<T> clazz, final int id, final SerializerShim<T> serializerShim) {
return new GryoTypeReg<>(clazz, null, serializerShim, null, id);
}
- private static <T> GryoTypeReg<T> of(Class clazz, int id, Function<Kryo, Serializer> fct) {
+ private static <T> GryoTypeReg<T> of(final Class clazz, final int id, final Function<Kryo, Serializer> fct) {
return new GryoTypeReg<>(clazz, null, null, fct, id);
}
@@ -606,7 +606,7 @@ public final class GryoMapper implements Mapper<Kryo> {
}
@Override
- public Kryo registerWith(Kryo kryo) {
+ public Kryo registerWith(final Kryo kryo) {
if (null != functionOfShadedKryo)
kryo.register(clazz, functionOfShadedKryo.apply(kryo), id);
else if (null != shadedSerializer)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
index 2042a4a..da4fd7a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializers.java
@@ -49,12 +49,12 @@ public final class GryoSerializers {
*/
public final static class EdgeSerializer implements SerializerShim<Edge> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Edge edge) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Edge edge) {
kryo.writeClassAndObject(output, DetachedFactory.detach(edge, true));
}
@Override
- public <I extends InputShim> Edge read(KryoShim<I, ?> kryo, I input, Class<Edge> edgeClass) {
+ public <I extends InputShim> Edge read(final KryoShim<I, ?> kryo, final I input, final Class<Edge> edgeClass) {
final Object o = kryo.readClassAndObject(input);
return (Edge) o;
}
@@ -65,12 +65,12 @@ public final class GryoSerializers {
*/
public final static class VertexSerializer implements SerializerShim<Vertex> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Vertex vertex) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Vertex vertex) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertex, true));
}
@Override
- public <I extends InputShim> Vertex read(KryoShim<I, ?> kryo, I input, Class<Vertex> vertexClass) {
+ public <I extends InputShim> Vertex read(final KryoShim<I, ?> kryo, final I input, final Class<Vertex> vertexClass) {
return (Vertex) kryo.readClassAndObject(input);
}
}
@@ -80,12 +80,12 @@ public final class GryoSerializers {
*/
public final static class PropertySerializer implements SerializerShim<Property> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Property property) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Property property) {
kryo.writeClassAndObject(output, property instanceof VertexProperty ? DetachedFactory.detach((VertexProperty) property, true) : DetachedFactory.detach(property));
}
@Override
- public <I extends InputShim> Property read(KryoShim<I, ?> kryo, I input, Class<Property> propertyClass) {
+ public <I extends InputShim> Property read(final KryoShim<I, ?> kryo, final I input, final Class<Property> propertyClass) {
return (Property) kryo.readClassAndObject(input);
}
}
@@ -95,12 +95,12 @@ public final class GryoSerializers {
*/
public final static class VertexPropertySerializer implements SerializerShim<VertexProperty> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, VertexProperty vertexProperty) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final VertexProperty vertexProperty) {
kryo.writeClassAndObject(output, DetachedFactory.detach(vertexProperty, true));
}
@Override
- public <I extends InputShim> VertexProperty read(KryoShim<I, ?> kryo, I input, Class<VertexProperty> vertexPropertyClass) {
+ public <I extends InputShim> VertexProperty read(final KryoShim<I, ?> kryo, final I input, final Class<VertexProperty> vertexPropertyClass) {
return (VertexProperty) kryo.readClassAndObject(input);
}
}
@@ -110,12 +110,12 @@ public final class GryoSerializers {
*/
public final static class PathSerializer implements SerializerShim<Path> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Path path) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Path path) {
kryo.writeClassAndObject(output, DetachedFactory.detach(path, false));
}
@Override
- public <I extends InputShim> Path read(KryoShim<I, ?> kryo, I input, Class<Path> pathClass) {
+ public <I extends InputShim> Path read(final KryoShim<I, ?> kryo, final I input, final Class<Path> pathClass) {
return (Path) kryo.readClassAndObject(input);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
index 24df032..85bb3c0 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/JavaTimeSerializers.java
@@ -50,12 +50,12 @@ final class JavaTimeSerializers {
*/
final static class DurationSerializer implements SerializerShim<Duration> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Duration duration) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Duration duration) {
output.writeLong(duration.toNanos());
}
@Override
- public <I extends InputShim> Duration read(KryoShim<I, ?> kryo, I input, Class<Duration> durationClass) {
+ public <I extends InputShim> Duration read(final KryoShim<I, ?> kryo, final I input, final Class<Duration> durationClass) {
return Duration.ofNanos(input.readLong());
}
}
@@ -65,13 +65,13 @@ final class JavaTimeSerializers {
*/
final static class InstantSerializer implements SerializerShim<Instant> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Instant instant) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Instant instant) {
output.writeLong(instant.getEpochSecond());
output.writeInt(instant.getNano());
}
@Override
- public <I extends InputShim> Instant read(KryoShim<I, ?> kryo, I input, Class<Instant> aClass) {
+ public <I extends InputShim> Instant read(final KryoShim<I, ?> kryo, final I input, final Class<Instant> aClass) {
return Instant.ofEpochSecond(input.readLong(), input.readInt());
}
}
@@ -81,12 +81,12 @@ final class JavaTimeSerializers {
*/
final static class LocalDateSerializer implements SerializerShim<LocalDate> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, LocalDate localDate) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final LocalDate localDate) {
output.writeLong(localDate.toEpochDay());
}
@Override
- public <I extends InputShim> LocalDate read(KryoShim<I, ?> kryo, I input, Class<LocalDate> clazz) {
+ public <I extends InputShim> LocalDate read(final KryoShim<I, ?> kryo, final I input, final Class<LocalDate> clazz) {
return LocalDate.ofEpochDay(input.readLong());
}
}
@@ -96,7 +96,7 @@ final class JavaTimeSerializers {
*/
final static class LocalDateTimeSerializer implements SerializerShim<LocalDateTime> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, LocalDateTime localDateTime) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final LocalDateTime localDateTime) {
output.writeInt(localDateTime.getYear());
output.writeInt(localDateTime.getMonthValue());
output.writeInt(localDateTime.getDayOfMonth());
@@ -107,7 +107,7 @@ final class JavaTimeSerializers {
}
@Override
- public <I extends InputShim> LocalDateTime read(KryoShim<I, ?> kryo, I input, Class<LocalDateTime> clazz) {
+ public <I extends InputShim> LocalDateTime read(final KryoShim<I, ?> kryo, final I input, final Class<LocalDateTime> clazz) {
return LocalDateTime.of(input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt());
}
}
@@ -117,12 +117,12 @@ final class JavaTimeSerializers {
*/
final static class LocalTimeSerializer implements SerializerShim<LocalTime> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, LocalTime localTime) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final LocalTime localTime) {
output.writeLong(localTime.toNanoOfDay());
}
@Override
- public <I extends InputShim> LocalTime read(KryoShim<I, ?> kryo, I input, Class<LocalTime> clazz) {
+ public <I extends InputShim> LocalTime read(final KryoShim<I, ?> kryo, final I input, final Class<LocalTime> clazz) {
return LocalTime.ofNanoOfDay(input.readLong());
}
}
@@ -132,13 +132,13 @@ final class JavaTimeSerializers {
*/
final static class MonthDaySerializer implements SerializerShim<MonthDay> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, MonthDay monthDay) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final MonthDay monthDay) {
output.writeInt(monthDay.getMonthValue());
output.writeInt(monthDay.getDayOfMonth());
}
@Override
- public <I extends InputShim> MonthDay read(KryoShim<I, ?> kryo, I input, Class<MonthDay> clazz) {
+ public <I extends InputShim> MonthDay read(final KryoShim<I, ?> kryo, final I input, final Class<MonthDay> clazz) {
return MonthDay.of(input.readInt(), input.readInt());
}
}
@@ -148,13 +148,13 @@ final class JavaTimeSerializers {
*/
final static class OffsetDateTimeSerializer implements SerializerShim<OffsetDateTime> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, OffsetDateTime offsetDateTime) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final OffsetDateTime offsetDateTime) {
kryo.writeObject(output, offsetDateTime.toLocalDateTime());
kryo.writeObject(output, offsetDateTime.getOffset());
}
@Override
- public <I extends InputShim> OffsetDateTime read(KryoShim<I, ?> kryo, I input, Class<OffsetDateTime> clazz) {
+ public <I extends InputShim> OffsetDateTime read(final KryoShim<I, ?> kryo, final I input, final Class<OffsetDateTime> clazz) {
return OffsetDateTime.of(kryo.readObject(input, LocalDateTime.class), kryo.readObject(input, ZoneOffset.class));
}
}
@@ -164,13 +164,13 @@ final class JavaTimeSerializers {
*/
final static class OffsetTimeSerializer implements SerializerShim<OffsetTime> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, OffsetTime offsetTime) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final OffsetTime offsetTime) {
kryo.writeObject(output, offsetTime.toLocalTime());
kryo.writeObject(output, offsetTime.getOffset());
}
@Override
- public <I extends InputShim> OffsetTime read(KryoShim<I, ?> kryo, I input, Class<OffsetTime> clazz) {
+ public <I extends InputShim> OffsetTime read(final KryoShim<I, ?> kryo, final I input, final Class<OffsetTime> clazz) {
return OffsetTime.of(kryo.readObject(input, LocalTime.class), kryo.readObject(input, ZoneOffset.class));
}
}
@@ -180,14 +180,14 @@ final class JavaTimeSerializers {
*/
final static class PeriodSerializer implements SerializerShim<Period> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Period period) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Period period) {
output.writeInt(period.getYears());
output.writeInt(period.getMonths());
output.writeInt(period.getDays());
}
@Override
- public <I extends InputShim> Period read(KryoShim<I, ?> kryo, I input, Class<Period> clazz) {
+ public <I extends InputShim> Period read(final KryoShim<I, ?> kryo, final I input, final Class<Period> clazz) {
return Period.of(input.readInt(), input.readInt(), input.readInt());
}
}
@@ -197,12 +197,12 @@ final class JavaTimeSerializers {
*/
final static class YearSerializer implements SerializerShim<Year> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Year year) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Year year) {
output.writeInt(year.getValue());
}
@Override
- public <I extends InputShim> Year read(KryoShim<I, ?> kryo, I input, Class<Year> clazz) {
+ public <I extends InputShim> Year read(final KryoShim<I, ?> kryo, final I input, final Class<Year> clazz) {
return Year.of(input.readInt());
}
}
@@ -212,13 +212,13 @@ final class JavaTimeSerializers {
*/
final static class YearMonthSerializer implements SerializerShim<YearMonth> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, YearMonth monthDay) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final YearMonth monthDay) {
output.writeInt(monthDay.getYear());
output.writeInt(monthDay.getMonthValue());
}
@Override
- public <I extends InputShim> YearMonth read(KryoShim<I, ?> kryo, I input, Class<YearMonth> clazz) {
+ public <I extends InputShim> YearMonth read(final KryoShim<I, ?> kryo, final I input, final Class<YearMonth> clazz) {
return YearMonth.of(input.readInt(), input.readInt());
}
}
@@ -228,7 +228,7 @@ final class JavaTimeSerializers {
*/
final static class ZonedDateTimeSerializer implements SerializerShim<ZonedDateTime> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, ZonedDateTime zonedDateTime) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ZonedDateTime zonedDateTime) {
output.writeInt(zonedDateTime.getYear());
output.writeInt(zonedDateTime.getMonthValue());
output.writeInt(zonedDateTime.getDayOfMonth());
@@ -240,7 +240,7 @@ final class JavaTimeSerializers {
}
@Override
- public <I extends InputShim> ZonedDateTime read(KryoShim<I, ?> kryo, I input, Class<ZonedDateTime> clazz) {
+ public <I extends InputShim> ZonedDateTime read(final KryoShim<I, ?> kryo, final I input, final Class<ZonedDateTime> clazz) {
return ZonedDateTime.of(input.readInt(), input.readInt(), input.readInt(),
input.readInt(), input.readInt(), input.readInt(), input.readInt(),
ZoneId.of(input.readString()));
@@ -252,12 +252,12 @@ final class JavaTimeSerializers {
*/
final static class ZoneOffsetSerializer implements SerializerShim<ZoneOffset> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, ZoneOffset zoneOffset) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ZoneOffset zoneOffset) {
output.writeString(zoneOffset.getId());
}
@Override
- public <I extends InputShim> ZoneOffset read(KryoShim<I, ?> kryo, I input, Class<ZoneOffset> clazz) {
+ public <I extends InputShim> ZoneOffset read(final KryoShim<I, ?> kryo, final I input, final Class<ZoneOffset> clazz) {
return ZoneOffset.of(input.readString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
index 0464b22..1465b24 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/PairSerializer.java
@@ -33,13 +33,13 @@ import org.javatuples.Pair;
*/
final class PairSerializer implements SerializerShim<Pair> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, Pair pair) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final Pair pair) {
kryo.writeClassAndObject(output, pair.getValue0());
kryo.writeClassAndObject(output, pair.getValue1());
}
@Override
- public <I extends InputShim> Pair read(KryoShim<I, ?> kryo, I input, Class<Pair> pairClass) {
+ public <I extends InputShim> Pair read(final KryoShim<I, ?> kryo, final I input, final Class<Pair> pairClass) {
return Pair.with(kryo.readClassAndObject(input), kryo.readClassAndObject(input));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
index 1f41c0d..a01d967 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
@@ -66,7 +66,7 @@ public interface TypeRegistration<T> {
* @param kryo Kryo instance into which this type is registered
* @return the sole parameter
*/
- Kryo registerWith(Kryo kryo);
+ Kryo registerWith(final Kryo kryo);
/**
* Returns true if at least one of {@link #getShadedSerializer()}, {@link #getSerializerShim()}, or
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java
index 23bec16..424b1bf 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java
@@ -24,15 +24,15 @@ package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
*/
public interface InputShim {
- byte readByte();
+ public byte readByte();
- byte[] readBytes(int size);
+ public byte[] readBytes(final int size);
- String readString();
+ public String readString();
- long readLong();
+ public long readLong();
- int readInt();
+ public int readInt();
- double readDouble();
+ public double readDouble();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java
index a0f3f87..a4eb2cb 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShim.java
@@ -27,15 +27,15 @@ package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
*/
public interface KryoShim<I extends InputShim, O extends OutputShim> {
- <T> T readObject(I input, Class<T> type);
+ public <T> T readObject(final I input, final Class<T> type);
- Object readClassAndObject(I input);
+ public Object readClassAndObject(final I input);
- void writeObject(O output, Object object);
+ public void writeObject(final O output, final Object object);
- void writeClassAndObject(O output, Object object);
+ public void writeClassAndObject(final O output, final Object object);
- <T> T readObjectOrNull(I input, Class<T> type);
+ public <T> T readObjectOrNull(final I input, final Class<T> type);
- void writeObjectOrNull(O output, Object object, Class type);
+ public void writeObjectOrNull(final O output, final Object object, final Class type);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
index 7783856..ee0c8b3 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimService.java
@@ -52,7 +52,7 @@ public interface KryoShimService {
* @param source the stream from which to read an object's serialized form
* @return the first deserialized object available from {@code source}
*/
- Object readClassAndObject(InputStream source);
+ public Object readClassAndObject(final InputStream source);
/**
* Serializes an object to an output stream. This may flush the output stream.
@@ -60,7 +60,7 @@ public interface KryoShimService {
* @param o the object to serialize
* @param sink the stream into which the serialized object is written
*/
- void writeClassAndObject(Object o, OutputStream sink);
+ public void writeClassAndObject(final Object o, final OutputStream sink);
/**
* Returns this service's relative priority number. Unless explicitly overridden through a
@@ -81,7 +81,7 @@ public interface KryoShimService {
*
* @return this implementation's priority value
*/
- int getPriority();
+ public int getPriority();
/**
* Attempt to incorporate the supplied configuration in future read/write calls.
@@ -95,5 +95,5 @@ public interface KryoShimService {
*
* @param conf the configuration to apply to this service's internal serializer
*/
- void applyConfiguration(Configuration conf);
+ public void applyConfiguration(final Configuration conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
index fd57a3c..d16b1a5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -49,7 +49,7 @@ public class KryoShimServiceLoader {
*/
public static final String SHIM_CLASS_SYSTEM_PROPERTY = "tinkerpop.kryo.shim";
- public static void applyConfiguration(Configuration conf) {
+ public static void applyConfiguration(final Configuration conf) {
KryoShimServiceLoader.conf = conf;
load(true);
}
@@ -64,15 +64,15 @@ public class KryoShimServiceLoader {
* before selecting a new service to return
* @return the shim service
*/
- public static KryoShimService load(boolean forceReload) {
+ public static KryoShimService load(final boolean forceReload) {
if (null != cachedShimService && !forceReload) {
return cachedShimService;
}
- ArrayList<KryoShimService> services = new ArrayList<>();
+ final ArrayList<KryoShimService> services = new ArrayList<>();
- ServiceLoader<KryoShimService> sl = ServiceLoader.load(KryoShimService.class);
+ final ServiceLoader<KryoShimService> sl = ServiceLoader.load(KryoShimService.class);
KryoShimService result = null;
@@ -117,7 +117,7 @@ public class KryoShimServiceLoader {
log.info("Set {} provider to {} ({}) because its priority value ({}) is the highest available",
KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
- Configuration userConf = conf;
+ final Configuration userConf = conf;
if (null != userConf) {
log.info("Configuring {} provider {} with user-provided configuration",
@@ -145,10 +145,10 @@ public class KryoShimServiceLoader {
* @param o an object for which the instance and class are serialized
* @return the serialized form
*/
- public static byte[] writeClassAndObjectToBytes(Object o) {
- KryoShimService shimService = load();
+ public static byte[] writeClassAndObjectToBytes(final Object o) {
+ final KryoShimService shimService = load();
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
shimService.writeClassAndObject(o, baos);
@@ -163,8 +163,8 @@ public class KryoShimServiceLoader {
* @param <T> the type to which the deserialized object is cast as it is returned
* @return the deserialized object
*/
- public static <T> T readClassAndObject(InputStream source) {
- KryoShimService shimService = load();
+ public static <T> T readClassAndObject(final InputStream source) {
+ final KryoShimService shimService = load();
return (T)shimService.readClassAndObject(source);
}
@@ -183,16 +183,16 @@ public class KryoShimServiceLoader {
INSTANCE;
@Override
- public int compare(KryoShimService a, KryoShimService b) {
- int ap = a.getPriority();
- int bp = b.getPriority();
+ public int compare(final KryoShimService a, final KryoShimService b) {
+ final int ap = a.getPriority();
+ final int bp = b.getPriority();
if (ap < bp) {
return -1;
} else if (bp < ap) {
return 1;
} else {
- int result = a.getClass().getCanonicalName().compareTo(b.getClass().getCanonicalName());
+ final int result = a.getClass().getCanonicalName().compareTo(b.getClass().getCanonicalName());
if (0 == result) {
log.warn("Found two {} implementations with the same canonical classname: {}. " +
@@ -201,7 +201,7 @@ public class KryoShimServiceLoader {
"META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService.",
a.getClass().getCanonicalName());
} else {
- String winner = 0 < result ? a.getClass().getCanonicalName() : b.getClass().getCanonicalName();
+ final String winner = 0 < result ? a.getClass().getCanonicalName() : b.getClass().getCanonicalName();
log.warn("{} implementations {} and {} are tied with priority value {}. " +
"Preferring {} to the other because it has a lexicographically greater classname. " +
"Consider setting the system property \"{}\" instead of relying on priority tie-breaking.",
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java
index e4ca3d5..e3647f3 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java
@@ -24,17 +24,17 @@ package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
*/
public interface OutputShim {
- void writeByte(byte b);
+ public void writeByte(final byte b);
- void writeBytes(byte[] array, int offset, int count);
+ public void writeBytes(final byte[] array, final int offset, final int count);
- void writeString(String s);
+ public void writeString(final String s);
- void writeLong(long l);
+ public void writeLong(final long l);
- void writeInt(int i);
+ public void writeInt(final int i);
- void writeDouble(double d);
+ public void writeDouble(final double d);
- void flush();
+ public void flush();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
index e5f9005..c36cfee 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/SerializerShim.java
@@ -26,11 +26,11 @@ package org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim;
*/
public interface SerializerShim<T> {
- <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, T object);
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final T object);
- <I extends InputShim> T read(KryoShim<I, ?> kryo, I input, Class<T> clazz);
+ public <I extends InputShim> T read(final KryoShim<I, ?> kryo, final I input, final Class<T> clazz);
- default boolean isImmutable() {
+ public default boolean isImmutable() {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java
index d11c1c4..29524b9 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java
@@ -25,7 +25,7 @@ public class ShadedInputAdapter implements InputShim {
private final Input shadedInput;
- public ShadedInputAdapter(Input shadedInput) {
+ public ShadedInputAdapter(final Input shadedInput) {
this.shadedInput = shadedInput;
}
@@ -40,7 +40,7 @@ public class ShadedInputAdapter implements InputShim {
}
@Override
- public byte[] readBytes(int size) {
+ public byte[] readBytes(final int size) {
return shadedInput.readBytes(size);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java
index 4283298..5ce0f6d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedKryoAdapter.java
@@ -25,42 +25,42 @@ public class ShadedKryoAdapter implements KryoShim<ShadedInputAdapter, ShadedOut
private final Kryo shadedKryo;
- public ShadedKryoAdapter(Kryo shadedKryo) {
+ public ShadedKryoAdapter(final Kryo shadedKryo) {
this.shadedKryo = shadedKryo;
}
@Override
- public <T> T readObject(ShadedInputAdapter input, Class<T> type)
+ public <T> T readObject(final ShadedInputAdapter input, final Class<T> type)
{
return shadedKryo.readObject(input.getShadedInput(), type);
}
@Override
- public Object readClassAndObject(ShadedInputAdapter input)
+ public Object readClassAndObject(final ShadedInputAdapter input)
{
return shadedKryo.readClassAndObject(input.getShadedInput());
}
@Override
- public void writeObject(ShadedOutputAdapter output, Object object)
+ public void writeObject(final ShadedOutputAdapter output, final Object object)
{
shadedKryo.writeObject(output.getShadedOutput(), object);
}
@Override
- public void writeClassAndObject(ShadedOutputAdapter output, Object object)
+ public void writeClassAndObject(final ShadedOutputAdapter output, final Object object)
{
shadedKryo.writeClassAndObject(output.getShadedOutput(), object);
}
@Override
- public <T> T readObjectOrNull(ShadedInputAdapter input, Class<T> type)
+ public <T> T readObjectOrNull(final ShadedInputAdapter input, final Class<T> type)
{
return shadedKryo.readObjectOrNull(input.getShadedInput(), type);
}
@Override
- public void writeObjectOrNull(ShadedOutputAdapter output, Object object, Class type)
+ public void writeObjectOrNull(final ShadedOutputAdapter output, final Object object, final Class type)
{
shadedKryo.writeObjectOrNull(output.getShadedOutput(), object, type);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java
index 7547466..b05f528 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java
@@ -25,38 +25,38 @@ public class ShadedOutputAdapter implements OutputShim {
private final Output shadedOutput;
- public ShadedOutputAdapter(Output shadedOutput) {
+ public ShadedOutputAdapter(final Output shadedOutput) {
this.shadedOutput = shadedOutput;
}
@Override
- public void writeByte(byte b)
+ public void writeByte(final byte b)
{
shadedOutput.writeByte(b);
}
@Override
- public void writeBytes(byte[] array, int offset, int count) {
+ public void writeBytes(final byte[] array, final int offset, final int count) {
shadedOutput.writeBytes(array, offset, count);
}
@Override
- public void writeString(String s) {
+ public void writeString(final String s) {
shadedOutput.writeString(s);
}
@Override
- public void writeLong(long l) {
+ public void writeLong(final long l) {
shadedOutput.writeLong(l);
}
@Override
- public void writeInt(int i) {
+ public void writeInt(final int i) {
shadedOutput.writeInt(i);
}
@Override
- public void writeDouble(double d) {
+ public void writeDouble(final double d) {
shadedOutput.writeDouble(d);
}
@@ -65,8 +65,7 @@ public class ShadedOutputAdapter implements OutputShim {
shadedOutput.flush();
}
- Output getShadedOutput()
- {
+ Output getShadedOutput() {
return shadedOutput;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java
index 4ce27b1..28a44bd 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedSerializerAdapter.java
@@ -28,27 +28,27 @@ public class ShadedSerializerAdapter<T> extends Serializer<T> {
SerializerShim<T> serializer;
- public ShadedSerializerAdapter(SerializerShim<T> serializer) {
+ public ShadedSerializerAdapter(final SerializerShim<T> serializer) {
this.serializer = serializer;
setImmutable(this.serializer.isImmutable());
}
@Override
- public void write(Kryo kryo, Output output, T t) {
+ public void write(final Kryo kryo, final Output output, final T t) {
/* These adapters could be cached pretty efficiently in instance fields if it were guaranteed that this
* class was never subject to concurrent use. That's true of Kryo instances, but it is not clear that
* it is true of Serializer instances.
*/
- ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
- ShadedOutputAdapter shadedOutputAdapter = new ShadedOutputAdapter(output);
+ final ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
+ final ShadedOutputAdapter shadedOutputAdapter = new ShadedOutputAdapter(output);
serializer.write(shadedKryoAdapter, shadedOutputAdapter, t);
}
@Override
- public T read(Kryo kryo, Input input, Class<T> aClass) {
+ public T read(final Kryo kryo, final Input input, final Class<T> aClass) {
// Same caching opportunity as in write(...)
- ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
- ShadedInputAdapter shadedInputAdapter = new ShadedInputAdapter(input);
+ final ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
+ final ShadedInputAdapter shadedInputAdapter = new ShadedInputAdapter(input);
return serializer.read(shadedKryoAdapter, shadedInputAdapter, aClass);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
index 431e1eb..f4ffa7d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphSerializer.java
@@ -81,7 +81,7 @@ public class StarGraphSerializer implements SerializerShim<StarGraph> {
* If the returned {@link StarGraph} is null, that means that the {@link GraphFilter} filtered the vertex.
*/
@Override
- public <I extends InputShim> StarGraph read(KryoShim<I, ?> kryo, I input, Class<StarGraph> clazz) {
+ public <I extends InputShim> StarGraph read(final KryoShim<I, ?> kryo, final I input, final Class<StarGraph> clazz) {
final StarGraph starGraph = StarGraph.open();
input.readByte(); // version field ignored for now - for future use with backward compatibility
starGraph.edgeProperties = kryo.readObjectOrNull(input, HashMap.class);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
index 12b5d07..3fad4fd 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
@@ -29,7 +29,7 @@ import java.io.OutputStream;
public class HadoopPoolShimService implements KryoShimService {
- public Object readClassAndObject(InputStream source) {
+ public Object readClassAndObject(final InputStream source) {
Kryo k = null;
@@ -44,14 +44,14 @@ public class HadoopPoolShimService implements KryoShimService {
}
}
- public void writeClassAndObject(Object o, OutputStream sink) {
+ public void writeClassAndObject(final Object o, final OutputStream sink) {
Kryo k = null;
try {
k = HadoopPools.getGryoPool().takeKryo();
- Output output = new Output(sink);
+ final Output output = new Output(sink);
k.writeClassAndObject(output, o);
@@ -69,7 +69,7 @@ public class HadoopPoolShimService implements KryoShimService {
}
@Override
- public void applyConfiguration(Configuration conf) {
+ public void applyConfiguration(final Configuration conf) {
HadoopPools.initialize(conf);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
index 88f7ee1..0379ee6 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/ObjectWritable.java
@@ -63,13 +63,13 @@ public final class ObjectWritable<T> implements WritableComparable<ObjectWritabl
@Override
public void readFields(final DataInput input) throws IOException {
- ByteArrayInputStream bais = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
+ final ByteArrayInputStream bais = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
this.t = KryoShimServiceLoader.readClassAndObject(bais);
}
@Override
public void write(final DataOutput output) throws IOException {
- byte serialized[] = KryoShimServiceLoader.writeClassAndObjectToBytes(this.t);
+ final byte serialized[] = KryoShimServiceLoader.writeClassAndObjectToBytes(this.t);
WritableUtils.writeCompressedByteArray(output, serialized);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
index 2252ded..be09941 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/VertexWritable.java
@@ -61,13 +61,13 @@ public final class VertexWritable implements Writable, Serializable {
@Override
public void readFields(final DataInput input) throws IOException {
this.vertex = null;
- ByteArrayInputStream bais = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
+ final ByteArrayInputStream bais = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
this.vertex = ((StarGraph)KryoShimServiceLoader.readClassAndObject(bais)).getStarVertex(); // read the star graph;
}
@Override
public void write(final DataOutput output) throws IOException {
- byte serialized[] = KryoShimServiceLoader.writeClassAndObjectToBytes(this.vertex.graph());
+ final byte serialized[] = KryoShimServiceLoader.writeClassAndObjectToBytes(this.vertex.graph());
WritableUtils.writeCompressedByteArray(output, serialized);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
index 68112d7..412fd3b 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
@@ -74,11 +74,11 @@ public class GryoRegistrator implements KryoRegistrator {
* or if they would be registered by this class by default (does not affect Kryo's
* built-in registrations, e.g. String.class).
*/
- public void registerClasses(Kryo kryo, Map<Class<?>, Serializer<?>> serializerOverrides, Set<Class<?>> blacklist) {
+ public void registerClasses(final Kryo kryo, final Map<Class<?>, Serializer<?>> serializerOverrides, final Set<Class<?>> blacklist) {
// Apply TinkerPop type registrations copied from GyroSerializer's constructor
for (Map.Entry<Class<?>, Serializer<?>> ent : getExtraRegistrations().entrySet()) {
- Class<?> targetClass = ent.getKey();
- Serializer<?> ser = ent.getValue();
+ final Class<?> targetClass = ent.getKey();
+ final Serializer<?> ser = ent.getValue();
// Is this class blacklisted? Skip it. (takes precedence over serializerOverrides)
if (blacklist.contains(targetClass)) {
@@ -97,7 +97,7 @@ public class GryoRegistrator implements KryoRegistrator {
}
}
- Set<Class<?>> shimmedClassesFromGryoMapper = new HashSet<>();
+ final Set<Class<?>> shimmedClassesFromGryoMapper = new HashSet<>();
// Apply GryoMapper's default registrations
for (TypeRegistration<?> tr : GryoMapper.build().create().getTypeRegistrations()) {
@@ -171,7 +171,7 @@ public class GryoRegistrator implements KryoRegistrator {
* iteration in key-insertion-order).
*/
- LinkedHashMap<Class<?>, Serializer<?>> m = new LinkedHashMap<>();
+ final LinkedHashMap<Class<?>, Serializer<?>> m = new LinkedHashMap<>();
// The following entries were copied from GryoSerializer's constructor
// This could be turned into a static collection on GryoSerializer to avoid
// duplication, but it would be a bit cumbersome to do so without disturbing
@@ -218,10 +218,10 @@ public class GryoRegistrator implements KryoRegistrator {
return m;
}
- private boolean checkForAndApplySerializerOverride(Map<Class<?>, Serializer<?>> serializerOverrides,
- Kryo kryo, Class<?> targetClass) {
+ private boolean checkForAndApplySerializerOverride(final Map<Class<?>, Serializer<?>> serializerOverrides,
+ final Kryo kryo, Class<?> targetClass) {
if (serializerOverrides.containsKey(targetClass)) {
- Serializer<?> ser = serializerOverrides.get(targetClass);
+ final Serializer<?> ser = serializerOverrides.get(targetClass);
if (null == ser) {
// null means use Kryo's default serializer
log.debug("Registering {} with default serializer per overrides", targetClass);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java
index 8b21e21..bf71fae 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/IoRegistryAwareKryoSerializer.java
@@ -46,7 +46,7 @@ public class IoRegistryAwareKryoSerializer extends KryoSerializer {
private static final Logger log = LoggerFactory.getLogger(IoRegistryAwareKryoSerializer.class);
- public IoRegistryAwareKryoSerializer(SparkConf conf) {
+ public IoRegistryAwareKryoSerializer(final SparkConf conf) {
super(conf);
// store conf so that we can access its registry (if one is present) in newKryo()
this.conf = conf;
@@ -54,19 +54,19 @@ public class IoRegistryAwareKryoSerializer extends KryoSerializer {
@Override
public Kryo newKryo() {
- Kryo kryo = super.newKryo();
+ final Kryo kryo = super.newKryo();
return applyIoRegistryIfPresent(kryo);
}
- private Kryo applyIoRegistryIfPresent(Kryo kryo) {
+ private Kryo applyIoRegistryIfPresent(final Kryo kryo) {
if (!conf.contains(GryoPool.CONFIG_IO_REGISTRY)) {
log.info("SparkConf {} does not contain setting {}, skipping {} handling",
GryoPool.CONFIG_IO_REGISTRY, conf, IoRegistry.class.getCanonicalName());
return kryo;
}
- String registryClassnames = conf.get(GryoPool.CONFIG_IO_REGISTRY);
+ final String registryClassnames = conf.get(GryoPool.CONFIG_IO_REGISTRY);
for (String registryClassname : registryClassnames.split(",")) {
final IoRegistry registry;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
index 4ceb045..01be50d 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/ObjectWritableSerializer.java
@@ -35,13 +35,13 @@ import org.apache.tinkerpop.shaded.kryo.io.Output;
public final class ObjectWritableSerializer<T> implements SerializerShim<ObjectWritable<T>> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, ObjectWritable<T> starGraph) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ObjectWritable<T> starGraph) {
kryo.writeClassAndObject(output, starGraph.get());
output.flush();
}
@Override
- public <I extends InputShim> ObjectWritable<T> read(KryoShim<I, ?> kryo, I input, Class<ObjectWritable<T>> clazz) {
+ public <I extends InputShim> ObjectWritable<T> read(final KryoShim<I, ?> kryo, final I input, final Class<ObjectWritable<T>> clazz) {
return new ObjectWritable(kryo.readClassAndObject(input));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
index f3c1b15..c89fb05 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/VertexWritableSerializer.java
@@ -36,13 +36,13 @@ import org.apache.tinkerpop.shaded.kryo.io.Output;
public final class VertexWritableSerializer implements SerializerShim<VertexWritable> {
@Override
- public <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, VertexWritable vertexWritable) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final VertexWritable vertexWritable) {
kryo.writeObject(output, vertexWritable.get().graph());
output.flush();
}
@Override
- public <I extends InputShim> VertexWritable read(KryoShim<I, ?> kryo, I input, Class<VertexWritable> clazz) {
+ public <I extends InputShim> VertexWritable read(final KryoShim<I, ?> kryo, final I input, final Class<VertexWritable> clazz) {
return new VertexWritable(kryo.readObject(input, StarGraph.class).getStarVertex());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java
index c533af7..0f13bb2 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java
@@ -32,7 +32,7 @@ public class UnshadedInputAdapter implements InputShim
private final Input unshadedInput;
- public UnshadedInputAdapter(Input unshadedInput)
+ public UnshadedInputAdapter(final Input unshadedInput)
{
this.unshadedInput = unshadedInput;
}
@@ -49,7 +49,7 @@ public class UnshadedInputAdapter implements InputShim
}
@Override
- public byte[] readBytes(int size) {
+ public byte[] readBytes(final int size) {
return unshadedInput.readBytes(size);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java
index b14abe0..94918a3 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoAdapter.java
@@ -31,43 +31,43 @@ public class UnshadedKryoAdapter implements KryoShim<UnshadedInputAdapter, Unsha
{
private final Kryo unshadedKryo;
- public UnshadedKryoAdapter(Kryo unshadedKryo)
+ public UnshadedKryoAdapter(final Kryo unshadedKryo)
{
this.unshadedKryo = unshadedKryo;
}
@Override
- public <T> T readObject(UnshadedInputAdapter input, Class<T> type)
+ public <T> T readObject(final UnshadedInputAdapter input, final Class<T> type)
{
return unshadedKryo.readObject(input.getUnshadedInput(), type);
}
@Override
- public Object readClassAndObject(UnshadedInputAdapter input)
+ public Object readClassAndObject(final UnshadedInputAdapter input)
{
return unshadedKryo.readClassAndObject(input.getUnshadedInput());
}
@Override
- public void writeObject(UnshadedOutputAdapter output, Object object)
+ public void writeObject(final UnshadedOutputAdapter output, final Object object)
{
unshadedKryo.writeObject(output.getUnshadedOutput(), object);
}
@Override
- public void writeClassAndObject(UnshadedOutputAdapter output, Object object)
+ public void writeClassAndObject(final UnshadedOutputAdapter output, final Object object)
{
unshadedKryo.writeClassAndObject(output.getUnshadedOutput(), object);
}
@Override
- public <T> T readObjectOrNull(UnshadedInputAdapter input, Class<T> type)
+ public <T> T readObjectOrNull(final UnshadedInputAdapter input, final Class<T> type)
{
return unshadedKryo.readObjectOrNull(input.getUnshadedInput(), type);
}
@Override
- public void writeObjectOrNull(UnshadedOutputAdapter output, Object object, Class type)
+ public void writeObjectOrNull(final UnshadedOutputAdapter output, final Object object, final Class type)
{
unshadedKryo.writeObjectOrNull(output.getUnshadedOutput(), object, type);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
index 95ebb0d..41e0001 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedKryoShimService.java
@@ -51,9 +51,9 @@ public class UnshadedKryoShimService implements KryoShimService {
public UnshadedKryoShimService() { }
@Override
- public Object readClassAndObject(InputStream source) {
+ public Object readClassAndObject(final InputStream source) {
- LinkedBlockingQueue<Kryo> kryos = initialize();
+ final LinkedBlockingQueue<Kryo> kryos = initialize();
Kryo k = null;
try {
@@ -72,15 +72,15 @@ public class UnshadedKryoShimService implements KryoShimService {
}
@Override
- public void writeClassAndObject(Object o, OutputStream sink) {
+ public void writeClassAndObject(final Object o, OutputStream sink) {
- LinkedBlockingQueue<Kryo> kryos = initialize();
+ final LinkedBlockingQueue<Kryo> kryos = initialize();
Kryo k = null;
try {
k = kryos.take();
- Output kryoOutput = new Output(sink);
+ final Output kryoOutput = new Output(sink);
k.writeClassAndObject(kryoOutput, o);
@@ -102,7 +102,7 @@ public class UnshadedKryoShimService implements KryoShimService {
}
@Override
- public void applyConfiguration(Configuration conf) {
+ public void applyConfiguration(final Configuration conf) {
initialize(conf);
}
@@ -110,22 +110,22 @@ public class UnshadedKryoShimService implements KryoShimService {
return initialize(new BaseConfiguration());
}
- private LinkedBlockingQueue<Kryo> initialize(Configuration conf) {
+ private LinkedBlockingQueue<Kryo> initialize(final Configuration conf) {
// DCL is safe in this case due to volatility
if (!initialized) {
synchronized (UnshadedKryoShimService.class) {
if (!initialized) {
- SparkConf sparkConf = new SparkConf();
+ final SparkConf sparkConf = new SparkConf();
// Copy the user's IoRegistry from the param conf to the SparkConf we just created
- String regStr = conf.getString(GryoPool.CONFIG_IO_REGISTRY);
+ final String regStr = conf.getString(GryoPool.CONFIG_IO_REGISTRY);
if (null != regStr) { // SparkConf rejects null values with NPE, so this has to be checked before set(...)
sparkConf.set(GryoPool.CONFIG_IO_REGISTRY, regStr);
}
// Setting spark.serializer here almost certainly isn't necessary, but it doesn't hurt
sparkConf.set("spark.serializer", IoRegistryAwareKryoSerializer.class.getCanonicalName());
- String registrator = conf.getString("spark.kryo.registrator");
+ final String registrator = conf.getString("spark.kryo.registrator");
if (null != registrator) {
sparkConf.set("spark.kryo.registrator", registrator);
log.info("Copied spark.kryo.registrator: {}", registrator);
@@ -134,7 +134,7 @@ public class UnshadedKryoShimService implements KryoShimService {
}
// Reuse Gryo poolsize for Kryo poolsize (no need to copy this to SparkConf)
- int poolSize = conf.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
+ final int poolSize = conf.getInt(GryoPool.CONFIG_IO_GRYO_POOL_SIZE,
GryoPool.CONFIG_IO_GRYO_POOL_SIZE_DEFAULT);
// Instantiate the spark.serializer
final IoRegistryAwareKryoSerializer ioReg = new IoRegistryAwareKryoSerializer(sparkConf);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java
index 9cc59d4..8dd8eb3 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java
@@ -31,7 +31,7 @@ public class UnshadedOutputAdapter implements OutputShim
{
private final Output unshadedOutput;
- public UnshadedOutputAdapter(Output unshadedOutput)
+ public UnshadedOutputAdapter(final Output unshadedOutput)
{
this.unshadedOutput = unshadedOutput;
}
@@ -42,35 +42,35 @@ public class UnshadedOutputAdapter implements OutputShim
}
@Override
- public void writeByte(byte b)
+ public void writeByte(final byte b)
{
unshadedOutput.writeByte(b);
}
@Override
- public void writeBytes(byte[] array, int offset, int count) {
+ public void writeBytes(final byte[] array, final int offset, final int count) {
unshadedOutput.writeBytes(array, offset, count);
}
@Override
- public void writeString(String s)
+ public void writeString(final String s)
{
unshadedOutput.writeString(s);
}
@Override
- public void writeLong(long l)
+ public void writeLong(final long l)
{
unshadedOutput.writeLong(l);
}
@Override
- public void writeInt(int i) {
+ public void writeInt(final int i) {
unshadedOutput.writeInt(i);
}
@Override
- public void writeDouble(double d)
+ public void writeDouble(final double d)
{
unshadedOutput.writeDouble(d);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/45abdcc3/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java
index efc9a4f..a5f8b05 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedSerializerAdapter.java
@@ -35,20 +35,20 @@ public class UnshadedSerializerAdapter<T> extends Serializer<T>
SerializerShim<T> serializer;
- public UnshadedSerializerAdapter(SerializerShim<T> serializer) {
+ public UnshadedSerializerAdapter(final SerializerShim<T> serializer) {
this.serializer = serializer;
setImmutable(this.serializer.isImmutable());
}
@Override
- public void write(Kryo kryo, Output output, T t) {
+ public void write(final Kryo kryo, final Output output, final T t) {
UnshadedKryoAdapter shadedKryoAdapter = new UnshadedKryoAdapter(kryo);
UnshadedOutputAdapter shadedOutputAdapter = new UnshadedOutputAdapter(output);
serializer.write(shadedKryoAdapter, shadedOutputAdapter, t);
}
@Override
- public T read(Kryo kryo, Input input, Class<T> aClass)
+ public T read(final Kryo kryo, final Input input, final Class<T> aClass)
{
UnshadedKryoAdapter shadedKryoAdapter = new UnshadedKryoAdapter(kryo);
UnshadedInputAdapter shadedInputAdapter = new UnshadedInputAdapter(input);
[02/18] incubator-tinkerpop git commit: Introduce Kryo shim to
support serializer reuse
Posted by ok...@apache.org.
Introduce Kryo shim to support serializer reuse
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ef528697
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ef528697
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ef528697
Branch: refs/heads/master
Commit: ef52869788ebf5b8b825f78ef21e1d38423d9aa0
Parents: aa673db
Author: Dan LaRocque <da...@hopcount.org>
Authored: Thu Jun 2 02:09:29 2016 -0500
Committer: Dan LaRocque <da...@hopcount.org>
Committed: Thu Jun 2 03:09:29 2016 -0400
----------------------------------------------------------------------
.../gremlin/structure/io/gryo/GryoMapper.java | 403 ++++++++++++-------
.../structure/io/gryo/TypeRegistration.java | 70 ++++
.../structure/io/gryo/URISerializer.java | 23 +-
.../structure/io/gryo/UUIDSerializer.java | 20 +-
.../structure/io/kryoshim/InputShim.java | 37 ++
.../gremlin/structure/io/kryoshim/KryoShim.java | 40 ++
.../structure/io/kryoshim/KryoShimService.java | 83 ++++
.../io/kryoshim/KryoShimServiceLoader.java | 137 +++++++
.../structure/io/kryoshim/OutputShim.java | 41 ++
.../structure/io/kryoshim/SerializerShim.java | 35 ++
.../structure/io/kryoshim/package-info.java | 54 +++
.../io/kryoshim/shaded/ShadedInputAdapter.java | 66 +++
.../io/kryoshim/shaded/ShadedKryoAdapter.java | 67 +++
.../io/kryoshim/shaded/ShadedOutputAdapter.java | 72 ++++
.../shaded/ShadedSerializerAdapter.java | 54 +++
.../io/kryoshim/shaded/package-info.java | 25 ++
.../util/star/StarGraphGryoSerializer.java | 123 +-----
.../util/star/StarGraphSerializer.java | 140 +++++++
.../structure/io/HadoopPoolShimService.java | 69 ++++
.../hadoop/structure/io/HadoopPools.java | 1 +
.../hadoop/structure/io/ObjectWritable.java | 25 +-
.../hadoop/structure/io/VertexWritable.java | 38 +-
...remlin.structure.io.kryoshim.KryoShimService | 1 +
23 files changed, 1293 insertions(+), 331 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 9cae845..5bc71da 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
import org.apache.tinkerpop.gremlin.process.traversal.Contains;
@@ -56,6 +57,8 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.Mapper;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded.ShadedSerializerAdapter;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPath;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty;
@@ -69,6 +72,7 @@ import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexProp
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphGryoSerializer;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphSerializer;
import org.apache.tinkerpop.shaded.kryo.ClassResolver;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.KryoSerializable;
@@ -77,7 +81,6 @@ import org.apache.tinkerpop.shaded.kryo.serializers.JavaSerializer;
import org.apache.tinkerpop.shaded.kryo.util.DefaultStreamFactory;
import org.apache.tinkerpop.shaded.kryo.util.MapReferenceResolver;
import org.javatuples.Pair;
-import org.javatuples.Triplet;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -156,13 +159,13 @@ import java.util.stream.Collectors;
public final class GryoMapper implements Mapper<Kryo> {
public static final byte[] GIO = "gio".getBytes();
public static final byte[] HEADER = Arrays.copyOf(GIO, 16);
- private final List<Triplet<Class, Function<Kryo, Serializer>, Integer>> serializationList;
+ private final List<TypeRegistration<?>> typeRegistrations;
private final boolean registrationRequired;
private final boolean referenceTracking;
private final Supplier<ClassResolver> classResolver;
private GryoMapper(final Builder builder) {
- this.serializationList = builder.serializationList;
+ this.typeRegistrations = builder.typeRegistrations;
validate();
this.registrationRequired = builder.registrationRequired;
@@ -176,19 +179,17 @@ public final class GryoMapper implements Mapper<Kryo> {
kryo.addDefaultSerializer(Map.Entry.class, new EntrySerializer());
kryo.setRegistrationRequired(registrationRequired);
kryo.setReferences(referenceTracking);
-
- serializationList.forEach(p -> {
- final Function<Kryo, Serializer> serializer = p.getValue1();
- if (null == serializer)
- kryo.register(p.getValue0(), kryo.getDefaultSerializer(p.getValue0()), p.getValue2());
- else
- kryo.register(p.getValue0(), serializer.apply(kryo), p.getValue2());
- });
+ for (TypeRegistration tr : typeRegistrations)
+ tr.registerWith(kryo);
return kryo;
}
public List<Class> getRegisteredClasses() {
- return this.serializationList.stream().map(Triplet::getValue0).collect(Collectors.toList());
+ return this.typeRegistrations.stream().map(TypeRegistration::getTargetClass).collect(Collectors.toList());
+ }
+
+ public List<TypeRegistration<?>> getTypeRegistrations() {
+ return typeRegistrations;
}
public static Builder build() {
@@ -199,11 +200,11 @@ public final class GryoMapper implements Mapper<Kryo> {
final Set<Integer> duplicates = new HashSet<>();
final Set<Integer> ids = new HashSet<>();
- serializationList.forEach(t -> {
- if (!ids.contains(t.getValue2()))
- ids.add(t.getValue2());
+ typeRegistrations.forEach(t -> {
+ if (!ids.contains(t.getId()))
+ ids.add(t.getId());
else
- duplicates.add(t.getValue2());
+ duplicates.add(t.getId());
});
if (duplicates.size() > 0)
@@ -244,119 +245,119 @@ public final class GryoMapper implements Mapper<Kryo> {
* Note that the following are pre-registered boolean, Boolean, byte, Byte, char, Character, double, Double,
* int, Integer, float, Float, long, Long, short, Short, String, void.
*/
- private final List<Triplet<Class, Function<Kryo, Serializer>, Integer>> serializationList = new ArrayList<Triplet<Class, Function<Kryo, Serializer>, Integer>>() {{
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(byte[].class, null, 25));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(char[].class, null, 26));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(short[].class, null, 27));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(int[].class, null, 28));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(long[].class, null, 29));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(float[].class, null, 30));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(double[].class, null, 31));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(String[].class, null, 32));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Object[].class, null, 33));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ArrayList.class, null, 10));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(BigInteger.class, null, 34));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(BigDecimal.class, null, 35));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Calendar.class, null, 39));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Class.class, null, 41));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Collection.class, null, 37));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Collections.EMPTY_LIST.getClass(), null, 51));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Collections.EMPTY_MAP.getClass(), null, 52));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Collections.EMPTY_SET.getClass(), null, 53));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Collections.singleton(null).getClass(), null, 54));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Collections.singletonList(null).getClass(), null, 24));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Collections.singletonMap(null, null).getClass(), null, 23));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Contains.class, null, 49));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Currency.class, null, 40));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Date.class, null, 38));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Direction.class, null, 12));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(DetachedEdge.class, null, 21));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(DetachedVertexProperty.class, null, 20));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(DetachedProperty.class, null, 18));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(DetachedVertex.class, null, 19));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(DetachedPath.class, null, 60));
+ private final List<TypeRegistration<?>> typeRegistrations = new ArrayList<TypeRegistration<?>>() {{
+ add(GryoTypeReg.of(byte[].class, 25));
+ add(GryoTypeReg.of(char[].class, 26));
+ add(GryoTypeReg.of(short[].class, 27));
+ add(GryoTypeReg.of(int[].class, 28));
+ add(GryoTypeReg.of(long[].class, 29));
+ add(GryoTypeReg.of(float[].class, 30));
+ add(GryoTypeReg.of(double[].class, 31));
+ add(GryoTypeReg.of(String[].class, 32));
+ add(GryoTypeReg.of(Object[].class, 33));
+ add(GryoTypeReg.of(ArrayList.class, 10));
+ add(GryoTypeReg.of(BigInteger.class, 34));
+ add(GryoTypeReg.of(BigDecimal.class, 35));
+ add(GryoTypeReg.of(Calendar.class, 39));
+ add(GryoTypeReg.of(Class.class, 41));
+ add(GryoTypeReg.of(Collection.class, 37));
+ add(GryoTypeReg.of(Collections.EMPTY_LIST.getClass(), 51));
+ add(GryoTypeReg.of(Collections.EMPTY_MAP.getClass(), 52));
+ add(GryoTypeReg.of(Collections.EMPTY_SET.getClass(), 53));
+ add(GryoTypeReg.of(Collections.singleton(null).getClass(), 54));
+ add(GryoTypeReg.of(Collections.singletonList(null).getClass(), 24));
+ add(GryoTypeReg.of(Collections.singletonMap(null, null).getClass(), 23));
+ add(GryoTypeReg.of(Contains.class, 49));
+ add(GryoTypeReg.of(Currency.class, 40));
+ add(GryoTypeReg.of(Date.class, 38));
+ add(GryoTypeReg.of(Direction.class, 12));
+ add(GryoTypeReg.of(DetachedEdge.class, 21));
+ add(GryoTypeReg.of(DetachedVertexProperty.class, 20));
+ add(GryoTypeReg.of(DetachedProperty.class, 18));
+ add(GryoTypeReg.of(DetachedVertex.class, 19));
+ add(GryoTypeReg.of(DetachedPath.class, 60));
// skip 14
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(EnumSet.class, null, 46));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(HashMap.class, null, 11));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(HashMap.Entry.class, null, 16));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(HASH_MAP_NODE, null, 92));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(KryoSerializable.class, null, 36));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LinkedHashMap.class, null, 47));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LinkedHashSet.class, null, 71));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LinkedList.class, null, 116));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LINKED_HASH_MAP_ENTRY_CLASS, null, 15));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Locale.class, null, 22));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(StringBuffer.class, null, 43));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(StringBuilder.class, null, 44));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(T.class, null, 48));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(TimeZone.class, null, 42));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(TreeMap.class, null, 45));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(TreeSet.class, null, 50));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(UUID.class, kryo -> new UUIDSerializer(), 17));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(URI.class, kryo -> new URISerializer(), 72));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(VertexTerminator.class, null, 13));
-
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ReferenceEdge.class, null, 81));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ReferenceVertexProperty.class, null, 82));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ReferenceProperty.class, null, 83));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ReferenceVertex.class, null, 84));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ReferencePath.class, null, 85));
-
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(StarGraph.class, kryo -> StarGraphGryoSerializer.with(Direction.BOTH), 86));
-
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Edge.class, kryo -> new GryoSerializers.EdgeSerializer(), 65));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Vertex.class, kryo -> new GryoSerializers.VertexSerializer(), 66));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Property.class, kryo -> new GryoSerializers.PropertySerializer(), 67));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(VertexProperty.class, kryo -> new GryoSerializers.VertexPropertySerializer(), 68));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Path.class, kryo -> new GryoSerializers.PathSerializer(), 59));
+ add(GryoTypeReg.of(EnumSet.class, 46));
+ add(GryoTypeReg.of(HashMap.class, 11));
+ add(GryoTypeReg.of(HashMap.Entry.class, 16));
+ add(GryoTypeReg.of(HASH_MAP_NODE, 92));
+ add(GryoTypeReg.of(KryoSerializable.class, 36));
+ add(GryoTypeReg.of(LinkedHashMap.class, 47));
+ add(GryoTypeReg.of(LinkedHashSet.class, 71));
+ add(GryoTypeReg.of(LinkedList.class, 116));
+ add(GryoTypeReg.of(LINKED_HASH_MAP_ENTRY_CLASS, 15));
+ add(GryoTypeReg.of(Locale.class, 22));
+ add(GryoTypeReg.of(StringBuffer.class, 43));
+ add(GryoTypeReg.of(StringBuilder.class, 44));
+ add(GryoTypeReg.of(T.class, 48));
+ add(GryoTypeReg.of(TimeZone.class, 42));
+ add(GryoTypeReg.of(TreeMap.class, 45));
+ add(GryoTypeReg.of(TreeSet.class, 50));
+ add(GryoTypeReg.of(UUID.class, 17, new UUIDSerializer()));
+ add(GryoTypeReg.of(URI.class, 72, new URISerializer()));
+ add(GryoTypeReg.of(VertexTerminator.class, 13));
+
+ add(GryoTypeReg.of(ReferenceEdge.class, 81));
+ add(GryoTypeReg.of(ReferenceVertexProperty.class, 82));
+ add(GryoTypeReg.of(ReferenceProperty.class, 83));
+ add(GryoTypeReg.of(ReferenceVertex.class, 84));
+ add(GryoTypeReg.of(ReferencePath.class, 85));
+
+ add(GryoTypeReg.of(StarGraph.class, 86, new StarGraphSerializer(Direction.BOTH, new GraphFilter())));
+
+ add(GryoTypeReg.of(Edge.class, 65, new GryoSerializers.EdgeSerializer()));
+ add(GryoTypeReg.of(Vertex.class, 66, new GryoSerializers.VertexSerializer()));
+ add(GryoTypeReg.of(Property.class, 67, new GryoSerializers.PropertySerializer()));
+ add(GryoTypeReg.of(VertexProperty.class, 68, new GryoSerializers.VertexPropertySerializer()));
+ add(GryoTypeReg.of(Path.class, 59, new GryoSerializers.PathSerializer()));
// skip 55
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(B_O_Traverser.class, null, 75));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(O_Traverser.class, null, 76));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(B_LP_O_P_S_SE_SL_Traverser.class, null, 77));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(B_O_S_SE_SL_Traverser.class, null, 78));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(B_LP_O_S_SE_SL_Traverser.class, null, 87));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(O_OB_S_SE_SL_Traverser.class, null, 89));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LP_O_OB_S_SE_SL_Traverser.class, null, 90));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LP_O_OB_P_S_SE_SL_Traverser.class, null, 91));
-
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(TraverserSet.class, null, 58));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Tree.class, null, 61));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(HashSet.class, null, 62));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(BulkSet.class, null, 64));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MutableMetrics.class, null, 69));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ImmutableMetrics.class, null, 115));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(DefaultTraversalMetrics.class, null, 70));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MapMemory.class, null, 73));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MapReduce.NullObject.class, null, 74));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(AtomicLong.class, null, 79));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Pair.class, kryo -> new PairSerializer(), 88));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(TraversalExplanation.class, kryo -> new JavaSerializer(), 106));
-
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Duration.class, kryo -> new JavaTimeSerializers.DurationSerializer(), 93));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Instant.class, kryo -> new JavaTimeSerializers.InstantSerializer(), 94));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LocalDate.class, kryo -> new JavaTimeSerializers.LocalDateSerializer(), 95));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LocalDateTime.class, kryo -> new JavaTimeSerializers.LocalDateTimeSerializer(), 96));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(LocalTime.class, kryo -> new JavaTimeSerializers.LocalTimeSerializer(), 97));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MonthDay.class, kryo -> new JavaTimeSerializers.MonthDaySerializer(), 98));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(OffsetDateTime.class, kryo -> new JavaTimeSerializers.OffsetDateTimeSerializer(), 99));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(OffsetTime.class, kryo -> new JavaTimeSerializers.OffsetTimeSerializer(), 100));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Period.class, kryo -> new JavaTimeSerializers.PeriodSerializer(), 101));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Year.class, kryo -> new JavaTimeSerializers.YearSerializer(), 102));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(YearMonth.class, kryo -> new JavaTimeSerializers.YearMonthSerializer(), 103));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ZonedDateTime.class, kryo -> new JavaTimeSerializers.ZonedDateTimeSerializer(), 104));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ZoneOffset.class, kryo -> new JavaTimeSerializers.ZoneOffsetSerializer(), 105));
-
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(Operator.class, null, 107));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(FoldStep.FoldBiOperator.class, null, 108));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(GroupCountStep.GroupCountBiOperator.class, null, 109));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(GroupStep.GroupBiOperator.class, kryo -> new JavaSerializer(), 117)); // because they contain traversals
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MeanGlobalStep.MeanGlobalBiOperator.class, null, 110));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(MeanGlobalStep.MeanNumber.class, null, 111));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(TreeStep.TreeBiOperator.class, null, 112));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(GroupStepV3d0.GroupBiOperatorV3d0.class, null, 113));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(RangeGlobalStep.RangeBiOperator.class, null, 114));
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(OrderGlobalStep.OrderBiOperator.class, kryo -> new JavaSerializer(), 118)); // because they contain traversals
- add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(ProfileStep.ProfileBiOperator.class, null, 119)); // ***LAST ID***
+ add(GryoTypeReg.of(B_O_Traverser.class, 75));
+ add(GryoTypeReg.of(O_Traverser.class, 76));
+ add(GryoTypeReg.of(B_LP_O_P_S_SE_SL_Traverser.class, 77));
+ add(GryoTypeReg.of(B_O_S_SE_SL_Traverser.class, 78));
+ add(GryoTypeReg.of(B_LP_O_S_SE_SL_Traverser.class, 87));
+ add(GryoTypeReg.of(O_OB_S_SE_SL_Traverser.class, 89));
+ add(GryoTypeReg.of(LP_O_OB_S_SE_SL_Traverser.class, 90));
+ add(GryoTypeReg.of(LP_O_OB_P_S_SE_SL_Traverser.class, 91));
+
+ add(GryoTypeReg.of(TraverserSet.class, 58));
+ add(GryoTypeReg.of(Tree.class, 61));
+ add(GryoTypeReg.of(HashSet.class, 62));
+ add(GryoTypeReg.of(BulkSet.class, 64));
+ add(GryoTypeReg.of(MutableMetrics.class, 69));
+ add(GryoTypeReg.of(ImmutableMetrics.class, 115));
+ add(GryoTypeReg.of(DefaultTraversalMetrics.class, 70));
+ add(GryoTypeReg.of(MapMemory.class, 73));
+ add(GryoTypeReg.of(MapReduce.NullObject.class, 74));
+ add(GryoTypeReg.of(AtomicLong.class, 79));
+ add(GryoTypeReg.of(Pair.class, 88, new PairSerializer()));
+ add(GryoTypeReg.of(TraversalExplanation.class, 106, new JavaSerializer()));
+
+ add(GryoTypeReg.of(Duration.class, 93, new JavaTimeSerializers.DurationSerializer()));
+ add(GryoTypeReg.of(Instant.class, 94, new JavaTimeSerializers.InstantSerializer()));
+ add(GryoTypeReg.of(LocalDate.class, 95, new JavaTimeSerializers.LocalDateSerializer()));
+ add(GryoTypeReg.of(LocalDateTime.class, 96, new JavaTimeSerializers.LocalDateTimeSerializer()));
+ add(GryoTypeReg.of(LocalTime.class, 97, new JavaTimeSerializers.LocalTimeSerializer()));
+ add(GryoTypeReg.of(MonthDay.class, 98, new JavaTimeSerializers.MonthDaySerializer()));
+ add(GryoTypeReg.of(OffsetDateTime.class, 99, new JavaTimeSerializers.OffsetDateTimeSerializer()));
+ add(GryoTypeReg.of(OffsetTime.class, 100, new JavaTimeSerializers.OffsetTimeSerializer()));
+ add(GryoTypeReg.of(Period.class, 101, new JavaTimeSerializers.PeriodSerializer()));
+ add(GryoTypeReg.of(Year.class, 102, new JavaTimeSerializers.YearSerializer()));
+ add(GryoTypeReg.of(YearMonth.class, 103, new JavaTimeSerializers.YearMonthSerializer()));
+ add(GryoTypeReg.of(ZonedDateTime.class, 104, new JavaTimeSerializers.ZonedDateTimeSerializer()));
+ add(GryoTypeReg.of(ZoneOffset.class, 105, new JavaTimeSerializers.ZoneOffsetSerializer()));
+
+ add(GryoTypeReg.of(Operator.class, 107));
+ add(GryoTypeReg.of(FoldStep.FoldBiOperator.class, 108));
+ add(GryoTypeReg.of(GroupCountStep.GroupCountBiOperator.class, 109));
+ add(GryoTypeReg.of(GroupStep.GroupBiOperator.class, 117, new JavaSerializer())); // because they contain traversals
+ add(GryoTypeReg.of(MeanGlobalStep.MeanGlobalBiOperator.class, 110));
+ add(GryoTypeReg.of(MeanGlobalStep.MeanNumber.class, 111));
+ add(GryoTypeReg.of(TreeStep.TreeBiOperator.class, 112));
+ add(GryoTypeReg.of(GroupStepV3d0.GroupBiOperatorV3d0.class, 113));
+ add(GryoTypeReg.of(RangeGlobalStep.RangeBiOperator.class, 114));
+ add(GryoTypeReg.of(OrderGlobalStep.OrderBiOperator.class, 118, new JavaSerializer())); // because they contain traversals
+ add(GryoTypeReg.of(ProfileStep.ProfileBiOperator.class, 119)); // ***LAST ID***
}};
private final List<IoRegistry> registries = new ArrayList<>();
@@ -404,8 +405,8 @@ public final class GryoMapper implements Mapper<Kryo> {
*/
public Builder addCustom(final Class... custom) {
if (custom != null && custom.length > 0) {
- for (Class clazz : custom) {
- addCustom(clazz, (Function<Kryo, Serializer>) null);
+ for (Class c : custom) {
+ addOrOverrideRegistration(GryoTypeReg.of(c, currentSerializationId.getAndIncrement()));
}
}
return this;
@@ -416,10 +417,15 @@ public final class GryoMapper implements Mapper<Kryo> {
* a class that is already registered will override that registration.
*/
public Builder addCustom(final Class clazz, final Serializer serializer) {
- if (null == serializer)
- addCustom(clazz);
- else
- addCustom(clazz, kryo -> serializer);
+ addOrOverrideRegistration(GryoTypeReg.of(clazz, currentSerializationId.getAndIncrement(), serializer));
+ return this;
+ }
+
+ /**
+ * Register custom class to serialize with a custom serialization shim.
+ */
+ public Builder addCustom(final Class clazz, final SerializerShim serializer) {
+ addOrOverrideRegistration(GryoTypeReg.of(clazz, currentSerializationId.getAndIncrement(), serializer));
return this;
}
@@ -427,15 +433,8 @@ public final class GryoMapper implements Mapper<Kryo> {
* Register a custom class to serialize with a custom serializer as returned from a {@link Function}. Note
* that calling this method for a class that is already registered will override that registration.
*/
- public Builder addCustom(final Class clazz, final Function<Kryo, Serializer> serializer) {
- final Optional<Triplet<Class, Function<Kryo, Serializer>, Integer>> found = findSerializer(clazz);
- if (found.isPresent()) {
- final Triplet<Class, Function<Kryo, Serializer>, Integer> t = found.get();
- serializationList.remove(t);
- serializationList.add(t.setAt1(serializer));
- } else
- serializationList.add(Triplet.with(clazz, serializer, currentSerializationId.getAndIncrement()));
-
+ public Builder addCustom(final Class clazz, final Function<Kryo, Serializer> functionOfKryo) {
+ addOrOverrideRegistration(GryoTypeReg.of(clazz, currentSerializationId.getAndIncrement(), functionOfKryo));
return this;
}
@@ -481,23 +480,123 @@ public final class GryoMapper implements Mapper<Kryo> {
addCustom(p.getValue0(), (Function<Kryo, Serializer>) p.getValue1());
else
throw new IllegalStateException(String.format(
- "Unexpected value provided by the %s for %s - expects [null, %s implementation or Function<%s, %s>]",
- IoRegistry.class.getSimpleName(), p.getValue0().getClass().getSimpleName(),
+ "Unexpected value provided by %s for serializable class %s - expected a parameter in [null, %s implementation or Function<%s, %s>], but received %s",
+ registry.getClass().getSimpleName(), p.getValue0().getClass().getCanonicalName(),
Serializer.class.getName(), Kryo.class.getSimpleName(),
- Serializer.class.getSimpleName()));
+ Serializer.class.getSimpleName(), p.getValue1()));
});
});
return new GryoMapper(this);
}
- private Optional<Triplet<Class, Function<Kryo, Serializer>, Integer>> findSerializer(final Class clazz) {
- final Iterator<Triplet<Class, Function<Kryo, Serializer>, Integer>> itty = IteratorUtils.filter(
- serializationList, t -> t.getValue0().equals(clazz)).iterator();
- if (itty.hasNext())
- return Optional.of(itty.next());
- else
- return Optional.empty();
+ private <T> void addOrOverrideRegistration(TypeRegistration<T> newRegistration) {
+ Iterator<TypeRegistration<?>> iter = typeRegistrations.iterator();
+ while (iter.hasNext()) {
+ TypeRegistration<?> existingRegistration = iter.next();
+ if (existingRegistration.getTargetClass().equals(newRegistration.getTargetClass())) {
+ iter.remove();
+ break;
+ }
+ }
+ typeRegistrations.add(newRegistration);
+ }
+ }
+
+ private static class GryoTypeReg<T> implements TypeRegistration<T> {
+
+ private final Class<T> clazz;
+ private final Serializer<T> shadedSerializer;
+ private final SerializerShim<T> serializerShim;
+ private final Function<Kryo, Serializer> functionOfShadedKryo;
+ private final int id;
+
+ private GryoTypeReg(Class<T> clazz,
+ Serializer<T> shadedSerializer,
+ SerializerShim<T> serializerShim,
+ Function<Kryo, Serializer> functionOfShadedKryo,
+ int id) {
+ this.clazz = clazz;
+ this.shadedSerializer = shadedSerializer;
+ this.serializerShim = serializerShim;
+ this.functionOfShadedKryo = functionOfShadedKryo;
+ this.id = id;
+
+ int serializerCount = 0;
+ if (null != this.shadedSerializer)
+ serializerCount++;
+ if (null != this.serializerShim)
+ serializerCount++;
+ if (null != this.functionOfShadedKryo)
+ serializerCount++;
+
+ if (1 < serializerCount) {
+ String msg = String.format(
+ "GryoTypeReg accepts at most one kind of serializer, but multiple " +
+ "serializers were supplied for class %s (id %s). " +
+ "Shaded serializer: %s. Shim serializer: %s. Shaded serializer function: %s.",
+ this.clazz.getCanonicalName(), id,
+ this.shadedSerializer, this.serializerShim, this.functionOfShadedKryo);
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ private static <T> GryoTypeReg<T> of(Class<T> clazz, int id) {
+ return new GryoTypeReg<>(clazz, null, null, null, id);
+ }
+
+ private static <T> GryoTypeReg<T> of(Class<T> clazz, int id, Serializer<T> shadedSerializer) {
+ return new GryoTypeReg<>(clazz, shadedSerializer, null, null, id);
+ }
+
+ private static <T> GryoTypeReg<T> of(Class<T> clazz, int id, SerializerShim<T> serializerShim) {
+ return new GryoTypeReg<>(clazz, null, serializerShim, null, id);
+ }
+
+ private static <T> GryoTypeReg<T> of(Class clazz, int id, Function<Kryo, Serializer> fct) {
+ return new GryoTypeReg<>(clazz, null, null, fct, id);
+ }
+
+ @Override
+ public Serializer<T> getShadedSerializer() {
+ return shadedSerializer;
+ }
+
+ @Override
+ public SerializerShim<T> getSerializerShim() {
+ return serializerShim;
+ }
+
+ @Override
+ public Function<Kryo, Serializer> getFunctionOfShadedKryo() {
+ return functionOfShadedKryo;
+ }
+
+ @Override
+ public Class<T> getTargetClass() {
+ return clazz;
+ }
+
+ @Override
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public Kryo registerWith(Kryo kryo) {
+ if (null != functionOfShadedKryo)
+ kryo.register(clazz, functionOfShadedKryo.apply(kryo), id);
+ else if (null != shadedSerializer)
+ kryo.register(clazz, shadedSerializer, id);
+ else if (null != serializerShim)
+ kryo.register(clazz, new ShadedSerializerAdapter<>(serializerShim), id);
+ else {
+ kryo.register(clazz, kryo.getDefaultSerializer(clazz), id);
+ // Suprisingly, the preceding call is not equivalent to
+ // kryo.register(clazz, id);
+ }
+
+ return kryo;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
new file mode 100644
index 0000000..5ca3f31
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/TypeRegistration.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.gryo;
+
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+
+/**
+ * Represents a class serializable with Gryo.
+ * <p>
+ * At most one of the {@link #getShadedSerializer()}, {@link #getSerializerShim()},
+ * and {@link #getFunctionOfShadedKryo()} will return a non-null value. If all
+ * three methods return null, then there is no custom serialization logic associated
+ * with this class. Gryo/Kryo will use its default serializer.
+ *
+ * @param <T> the serializable type
+ */
+public interface TypeRegistration<T> {
+
+ /**
+ * @return the serializable class this instance describes
+ */
+ Class<T> getTargetClass();
+
+ /**
+ * @return numeric identifier used as a shorthand for this type in Gryo's serialized form
+ */
+ int getId();
+
+ /**
+ * @return the shaded-Kryo serializer that handles this type, if one is defined
+ */
+ Serializer<T> getShadedSerializer();
+
+ /**
+ * @return the shim-Kryo serializer that handles this type, if one is defined
+ */
+ SerializerShim<T> getSerializerShim();
+
+ /**
+ * @return a function that accepts a shaded-Kryo instance and returns a serializer, if such a function is defined
+ */
+ java.util.function.Function<Kryo, Serializer> getFunctionOfShadedKryo();
+
+ /**
+ * Registers this type on the supplied {@link Kryo} instance, using whatever custom serializer
+ * may be present, then returns the same {@linkplain Kryo} instance supplied as the parameter.
+ *
+ * @param kryo Kryo instance into which this type is registered
+ * @return the sole parameter
+ */
+ Kryo registerWith(Kryo kryo);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
index 7b6cfec..de08061 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/URISerializer.java
@@ -18,29 +18,32 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
import java.net.URI;
/**
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
-final class URISerializer extends Serializer<URI> {
+final class URISerializer implements SerializerShim<URI> {
- public URISerializer() {
- setImmutable(true);
- }
+ public URISerializer() { }
@Override
- public void write(final Kryo kryo, final Output output, final URI uri) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final URI uri) {
output.writeString(uri.toString());
}
@Override
- public URI read(final Kryo kryo, final Input input, final Class<URI> uriClass) {
+ public <I extends InputShim> URI read(final KryoShim<I, ?> kryo, final I input, final Class<URI> uriClass) {
return URI.create(input.readString());
}
+
+ @Override
+ public boolean isImmutable() {
+ return true;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
index e694e38..b86ddc2 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/UUIDSerializer.java
@@ -18,6 +18,10 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.gryo;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.Serializer;
import org.apache.tinkerpop.shaded.kryo.io.Input;
@@ -28,19 +32,23 @@ import java.util.UUID;
/**
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
-final class UUIDSerializer extends Serializer<UUID> {
- public UUIDSerializer() {
- setImmutable(true);
- }
+final class UUIDSerializer implements SerializerShim<UUID> {
+
+ public UUIDSerializer() { }
@Override
- public void write(final Kryo kryo, final Output output, final UUID uuid) {
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final UUID uuid) {
output.writeLong(uuid.getMostSignificantBits());
output.writeLong(uuid.getLeastSignificantBits());
}
@Override
- public UUID read(final Kryo kryo, final Input input, final Class<UUID> uuidClass) {
+ public <I extends InputShim> UUID read(final KryoShim<I, ?> kryo, final I input, final Class<UUID> uuidClass) {
return new UUID(input.readLong(), input.readLong());
}
+
+ @Override
+ public boolean isImmutable() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java
new file mode 100644
index 0000000..6d00884
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/InputShim.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.io.Input}-like abstraction.
+ */
+public interface InputShim {
+
+ byte readByte();
+
+ byte[] readBytes(int size);
+
+ String readString();
+
+ long readLong();
+
+ int readInt();
+
+ double readDouble();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java
new file mode 100644
index 0000000..e2a95e6
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShim.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.Kryo}-like abstraction.
+ *
+ * @param <I> this interface's complementary InputShim
+ * @param <O> this interface's complementary OutputShim
+ */
+public interface KryoShim<I extends InputShim, O extends OutputShim> {
+
+ <T> T readObject(I input, Class<T> type);
+
+ Object readClassAndObject(I input);
+
+ void writeObject(O output, Object object);
+
+ void writeClassAndObject(O output, Object object);
+
+ <T> T readObjectOrNull(I input, Class<T> type);
+
+ void writeObjectOrNull(O output, Object object, Class type);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java
new file mode 100644
index 0000000..024d40c
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimService.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * This interface exists to decouple HadoopPools from TinkerPop's shaded Kryo.
+ * <p>
+ * VertexWritable and ObjectWritable formerly implemented Serializable by
+ * resorting to statically-pooled shaded Kryo instances maintained by the HadoopPools class.
+ * This is awkward because those shaded Kryo instances require class registration by default.
+ * <p>
+ * Consider what happens with custom property datatypes reachable from the reference graph rooted at an ObjectWritable
+ * or VertexWritable instance. It is not enough for these property classes to merely implement
+ * Serializable, though one think that from skimming ObjectWritable/VertexWritable. Those classes
+ * must also register with TinkerPop's internal, shaded Kryo instances as maintained by HadoopPools,
+ * or else configure those instances to accept unregistered classes.
+ * Otherwise, TinkerPop's shaded Kryo will refuse to serialize those properties (even though
+ * they implement Serializable, and even though the user might think they are only using
+ * Java's standard Serialization mechanism!).
+ * <p>
+ * By hiding the mechanics of serialization behind this interface instead of hardcoding it in
+ * HadoopPools, the user can decide how to implement serialization for ObjectWritable/VertexWritable
+ * (and whatever other classes in TinkerPop decide to implement Serializable but then delegate
+ * all of the implementation details, like ObjectWritable/VertexWritable do now).
+ */
+public interface KryoShimService {
+
+ /**
+ * Deserializes an object from an input stream.
+ *
+ * @param source the stream from which to read an object's serialized form
+ * @return the first deserialized object available from {@code source}
+ */
+ Object readClassAndObject(InputStream source);
+
+ /**
+ * Serializes an object to an output stream. This may flush the output stream.
+ *
+ * @param o the object to serialize
+ * @param sink the stream into which the serialized object is written
+ */
+ void writeClassAndObject(Object o, OutputStream sink);
+
+ /**
+ * Returns this service's relative priority number. Unless explicitly overridden through a
+ * system property ({@link KryoShimServiceLoader#SHIM_CLASS_SYSTEM_PROPERTY}),
+ * the service implementation with the numerically highest priority will be used
+ * and all others ignored. In other words, the highest priority wins (in the absence of a
+ * system property override).
+ * <p>
+ * TinkerPop's current default implementation uses priority value zero.
+ * <p>
+ * Third-party implementations of this interface should (but are not technically required)
+ * to use a priority value with absolute value greater than 100.
+ * <p>
+ * The implementation currently breaks priority ties by lexicographical comparison of
+ * fully-qualified package-and-classname, but this tie-breaking behavior should be
+ * considered undefined and subject to future change. Ties are ignored if the service
+ * is explicitly set through the system property mentioned above.
+ *
+ * @return this implementation's priority value
+ */
+ int getPriority();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java
new file mode 100644
index 0000000..1d5413d
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/KryoShimServiceLoader.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.ServiceLoader;
+
+public class KryoShimServiceLoader {
+
+ private static volatile KryoShimService CACHED_SHIM_SERVICE;
+
+ private static final Logger log = LoggerFactory.getLogger(KryoShimServiceLoader.class);
+
+ /**
+ * Set this system property to the fully-qualified name of a {@link KryoShimService}
+ * package-and-classname to force it into service. Setting this property causes the
+ * priority-selection mechanism ({@link KryoShimService#getPriority()}) to be ignored.
+ */
+ public static final String SHIM_CLASS_SYSTEM_PROPERTY = "tinkerpop.kryo.shim";
+
+ public static KryoShimService load(boolean forceReload) {
+
+ if (null != CACHED_SHIM_SERVICE && !forceReload) {
+ return CACHED_SHIM_SERVICE;
+ }
+
+ ArrayList<KryoShimService> services = new ArrayList<>();
+
+ ServiceLoader<KryoShimService> sl = ServiceLoader.load(KryoShimService.class);
+
+ KryoShimService result = null;
+
+ synchronized (KryoShimServiceLoader.class) {
+ if (forceReload) {
+ sl.reload();
+ }
+
+ for (KryoShimService kss : sl) {
+ services.add(kss);
+ }
+ }
+
+ String shimClass = System.getProperty(SHIM_CLASS_SYSTEM_PROPERTY);
+
+ if (null != shimClass) {
+ for (KryoShimService kss : services) {
+ if (kss.getClass().getCanonicalName().equals(shimClass)) {
+ log.info("Set {} provider to {} ({}) from system property {}={}",
+ KryoShimService.class.getSimpleName(), kss, kss.getClass(),
+ SHIM_CLASS_SYSTEM_PROPERTY, shimClass);
+ result = kss;
+ }
+ }
+ } else {
+ Collections.sort(services, KryoShimServiceComparator.INSTANCE);
+
+ for (KryoShimService kss : services) {
+ log.debug("Found Kryo shim service class {} (priority {})", kss.getClass(), kss.getPriority());
+ }
+
+ if (0 != services.size()) {
+ result = services.get(services.size() - 1);
+ }
+ }
+
+
+ if (null == result) {
+ throw new IllegalStateException("Unable to load KryoShimService");
+ }
+
+ log.info("Set {} provider to {} ({}) because its priority value ({}) is the highest available",
+ KryoShimService.class.getSimpleName(), result, result.getClass(), result.getPriority());
+
+ return CACHED_SHIM_SERVICE = result;
+ }
+
+ public static KryoShimService load() {
+ return load(false);
+ }
+
+ public static byte[] writeClassAndObjectToBytes(Object o) {
+ KryoShimService shimService = load();
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ shimService.writeClassAndObject(o, baos);
+
+ return baos.toByteArray();
+ }
+
+ public static <T> T readClassAndObject(InputStream source) {
+ KryoShimService shimService = load();
+
+ return (T)shimService.readClassAndObject(source);
+ }
+
+ private enum KryoShimServiceComparator implements Comparator<KryoShimService> {
+ INSTANCE;
+
+ @Override
+ public int compare(KryoShimService a, KryoShimService b) {
+ int ap = a.getPriority();
+ int bp = b.getPriority();
+
+ if (ap < bp) {
+ return -1;
+ } else if (bp < ap) {
+ return 1;
+ } else {
+ return a.getClass().getCanonicalName().compareTo(b.getClass().getCanonicalName());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java
new file mode 100644
index 0000000..4468434
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/OutputShim.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
+
+import java.io.IOException;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.io.Output}-like abstraction.
+ */
+public interface OutputShim {
+
+ void writeByte(byte b);
+
+ void writeBytes(byte[] array, int offset, int count);
+
+ void writeString(String s);
+
+ void writeLong(long l);
+
+ void writeInt(int i);
+
+ void writeDouble(double d);
+
+ void flush();
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java
new file mode 100644
index 0000000..686350d
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/SerializerShim.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
+
+/**
+ * A minimal {@link org.apache.tinkerpop.shaded.kryo.Serializer}-like abstraction.
+ *
+ * @param <T> the class this serializer reads/writes from/to bytes.
+ */
+public interface SerializerShim<T> {
+
+ <O extends OutputShim> void write(KryoShim<?, O> kryo, O output, T starGraph);
+
+ <I extends InputShim> T read(KryoShim<I, ?> kryo, I input, Class<T> clazz);
+
+ default boolean isImmutable() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java
new file mode 100644
index 0000000..436f117
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/package-info.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Abstracts a minimal subset of Kryo types and methods.
+ * <p>
+ * Kryo is often shaded. For instance, TinkerPop's Gryo
+ * serializer relies on a shaded Kryo package.
+ * TinkerPop serializers written against a particular shaded
+ * Kryo package (or the unshaded Kryo package) are compatible
+ * only with that package. In contrast, TinkerPop serializers written
+ * against this abstraction can be used with any shaded or
+ * unshaded Kryo package, so long as the signatures and behavior
+ * of the methods in this package remain stable.
+ * <p>
+ * To show how this is useful, consider
+ * {@link org.apache.tinkerpop.gremlin.structure.util.star.StarGraphSerializer}.
+ * This class has logic unique to TinkerPop that performs
+ * efficient and forward-compatible serialization of
+ * {@link org.apache.tinkerpop.gremlin.structure.util.star.StarGraph}
+ * instances. It takes advantage of package-level visibility
+ * and the fact that it shares a package with its target,
+ * so it would be challenging to cleanly and naturally replicate
+ * (i.e. without package spoofing or runtime visibility overrides).
+ * By implementing
+ * {@link org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim}
+ * instead of, say, Gryo's shaded
+ * {@link org.apache.tinkerpop.shaded.kryo.Serializer},
+ * it can be used with anybody's Kryo package, regardless of whether
+ * that package is shaded or not. This lets third-parties reuse
+ * TinkerPop's efficient, internals-aware StarGraph serializer on
+ * their own serialization platform (and without altering
+ * TinkerPop's bytecode, let alone its source).
+ * <p>
+ * The number of types and methods in this
+ * package is deliberately small to reduce the likelihood of a
+ * new Kryo release introducing an incompatible change.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java
new file mode 100644
index 0000000..2872326
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedInputAdapter.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.InputShim;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+
+public class ShadedInputAdapter implements InputShim {
+
+ private final Input shadedInput;
+
+ public ShadedInputAdapter(Input shadedInput) {
+ this.shadedInput = shadedInput;
+ }
+
+ Input getShadedInput() {
+ return shadedInput;
+ }
+
+ @Override
+ public byte readByte()
+ {
+ return shadedInput.readByte();
+ }
+
+ @Override
+ public byte[] readBytes(int size) {
+ return shadedInput.readBytes(size);
+ }
+
+ @Override
+ public String readString() {
+ return shadedInput.readString();
+ }
+
+ @Override
+ public long readLong() {
+ return shadedInput.readLong();
+ }
+
+ @Override
+ public int readInt() {
+ return shadedInput.readInt();
+ }
+
+ @Override
+ public double readDouble() {
+ return shadedInput.readDouble();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java
new file mode 100644
index 0000000..0e85f6f
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedKryoAdapter.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.KryoShim;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+
+public class ShadedKryoAdapter implements KryoShim<ShadedInputAdapter, ShadedOutputAdapter> {
+
+ private final Kryo shadedKryo;
+
+ public ShadedKryoAdapter(Kryo shadedKryo) {
+ this.shadedKryo = shadedKryo;
+ }
+
+ @Override
+ public <T> T readObject(ShadedInputAdapter input, Class<T> type)
+ {
+ return shadedKryo.readObject(input.getShadedInput(), type);
+ }
+
+ @Override
+ public Object readClassAndObject(ShadedInputAdapter input)
+ {
+ return shadedKryo.readClassAndObject(input.getShadedInput());
+ }
+
+ @Override
+ public void writeObject(ShadedOutputAdapter output, Object object)
+ {
+ shadedKryo.writeObject(output.getShadedOutput(), object);
+ }
+
+ @Override
+ public void writeClassAndObject(ShadedOutputAdapter output, Object object)
+ {
+ shadedKryo.writeClassAndObject(output.getShadedOutput(), object);
+ }
+
+ @Override
+ public <T> T readObjectOrNull(ShadedInputAdapter input, Class<T> type)
+ {
+ return shadedKryo.readObjectOrNull(input.getShadedInput(), type);
+ }
+
+ @Override
+ public void writeObjectOrNull(ShadedOutputAdapter output, Object object, Class type)
+ {
+ shadedKryo.writeObjectOrNull(output.getShadedOutput(), object, type);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java
new file mode 100644
index 0000000..5ca0ecb
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedOutputAdapter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.OutputShim;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+
+public class ShadedOutputAdapter implements OutputShim {
+
+ private final Output shadedOutput;
+
+ public ShadedOutputAdapter(Output shadedOutput) {
+ this.shadedOutput = shadedOutput;
+ }
+
+ @Override
+ public void writeByte(byte b)
+ {
+ shadedOutput.writeByte(b);
+ }
+
+ @Override
+ public void writeBytes(byte[] array, int offset, int count) {
+ shadedOutput.writeBytes(array, offset, count);
+ }
+
+ @Override
+ public void writeString(String s) {
+ shadedOutput.writeString(s);
+ }
+
+ @Override
+ public void writeLong(long l) {
+ shadedOutput.writeLong(l);
+ }
+
+ @Override
+ public void writeInt(int i) {
+ shadedOutput.writeInt(i);
+ }
+
+ @Override
+ public void writeDouble(double d) {
+ shadedOutput.writeDouble(d);
+ }
+
+ @Override
+ public void flush() {
+ shadedOutput.flush();
+ }
+
+ Output getShadedOutput()
+ {
+ return shadedOutput;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java
new file mode 100644
index 0000000..ebfd0fb
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/ShadedSerializerAdapter.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
+
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.SerializerShim;
+import org.apache.tinkerpop.shaded.kryo.Kryo;
+import org.apache.tinkerpop.shaded.kryo.Serializer;
+import org.apache.tinkerpop.shaded.kryo.io.Input;
+import org.apache.tinkerpop.shaded.kryo.io.Output;
+
+public class ShadedSerializerAdapter<T> extends Serializer<T> {
+
+ SerializerShim<T> serializer;
+
+ public ShadedSerializerAdapter(SerializerShim<T> serializer) {
+ this.serializer = serializer;
+ setImmutable(this.serializer.isImmutable());
+ }
+
+ @Override
+ public void write(Kryo kryo, Output output, T t) {
+ /* These adapters could be cached pretty efficiently in instance fields if it were guaranteed that this
+ * class was never subject to concurrent use. That's true of Kryo instances, but it is not clear that
+ * it is true of Serializer instances.
+ */
+ ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
+ ShadedOutputAdapter shadedOutputAdapter = new ShadedOutputAdapter(output);
+ serializer.write(shadedKryoAdapter, shadedOutputAdapter, t);
+ }
+
+ @Override
+ public T read(Kryo kryo, Input input, Class<T> aClass) {
+ // Same caching opportunity as in write(...)
+ ShadedKryoAdapter shadedKryoAdapter = new ShadedKryoAdapter(kryo);
+ ShadedInputAdapter shadedInputAdapter = new ShadedInputAdapter(input);
+ return serializer.read(shadedKryoAdapter, shadedInputAdapter, aClass);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java
new file mode 100644
index 0000000..ea70003
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/kryoshim/shaded/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Implementations of the interfaces in
+ * {@link org.apache.tinkerpop.gremlin.structure.io.kryoshim}
+ * using TinkerPop's shaded copy of Kryo.
+ */
+package org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef528697/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
index 3805e2c..2acd770 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraphGryoSerializer.java
@@ -18,20 +18,13 @@
*/
package org.apache.tinkerpop.gremlin.structure.util.star;
-import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.T;
-import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.apache.tinkerpop.shaded.kryo.Kryo;
-import org.apache.tinkerpop.shaded.kryo.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import org.apache.tinkerpop.shaded.kryo.io.Output;
-
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.io.kryoshim.shaded.ShadedSerializerAdapter;
+
/**
* Kryo serializer for {@link StarGraph}. Implements an internal versioning capability for backward compatibility.
* The single byte at the front of the serialization stream denotes the version. That version can be used to choose
@@ -42,15 +35,10 @@ import java.util.Map;
* @author Marko A. Rodriguez (http://markorodriguez.com)
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
-public final class StarGraphGryoSerializer extends Serializer<StarGraph> {
+public final class StarGraphGryoSerializer extends ShadedSerializerAdapter<StarGraph> {
private static final Map<Direction, StarGraphGryoSerializer> CACHE = new HashMap<>();
- private final Direction edgeDirectionToSerialize;
- private GraphFilter graphFilter = new GraphFilter(); // will allow all vertices/edges
-
- private final static byte VERSION_1 = Byte.MIN_VALUE;
-
static {
CACHE.put(Direction.BOTH, new StarGraphGryoSerializer(Direction.BOTH));
CACHE.put(Direction.IN, new StarGraphGryoSerializer(Direction.IN));
@@ -58,8 +46,12 @@ public final class StarGraphGryoSerializer extends Serializer<StarGraph> {
CACHE.put(null, new StarGraphGryoSerializer(null));
}
+ private StarGraphGryoSerializer(final Direction edgeDirectionToSerialize, final GraphFilter graphFilter) {
+ super(new StarGraphSerializer(edgeDirectionToSerialize, graphFilter));
+ }
+
private StarGraphGryoSerializer(final Direction edgeDirectionToSerialize) {
- this.edgeDirectionToSerialize = edgeDirectionToSerialize;
+ this(edgeDirectionToSerialize, new GraphFilter());
}
/**
@@ -71,100 +63,7 @@ public final class StarGraphGryoSerializer extends Serializer<StarGraph> {
}
public static StarGraphGryoSerializer withGraphFilter(final GraphFilter graphFilter) {
- final StarGraphGryoSerializer serializer = new StarGraphGryoSerializer(Direction.BOTH);
- serializer.graphFilter = graphFilter.clone();
+ final StarGraphGryoSerializer serializer = new StarGraphGryoSerializer(Direction.BOTH, graphFilter.clone());
return serializer;
}
-
- @Override
- public void write(final Kryo kryo, final Output output, final StarGraph starGraph) {
- output.writeByte(VERSION_1);
- kryo.writeObjectOrNull(output, starGraph.edgeProperties, HashMap.class);
- kryo.writeObjectOrNull(output, starGraph.metaProperties, HashMap.class);
- kryo.writeClassAndObject(output, starGraph.starVertex.id);
- kryo.writeObject(output, starGraph.starVertex.label);
- writeEdges(kryo, output, starGraph, Direction.IN);
- writeEdges(kryo, output, starGraph, Direction.OUT);
- kryo.writeObject(output, null != starGraph.starVertex.vertexProperties);
- if (null != starGraph.starVertex.vertexProperties) {
- kryo.writeObject(output, starGraph.starVertex.vertexProperties.size());
- for (final Map.Entry<String, List<VertexProperty>> vertexProperties : starGraph.starVertex.vertexProperties.entrySet()) {
- kryo.writeObject(output, vertexProperties.getKey());
- kryo.writeObject(output, vertexProperties.getValue().size());
- for (final VertexProperty vertexProperty : vertexProperties.getValue()) {
- kryo.writeClassAndObject(output, vertexProperty.id());
- kryo.writeClassAndObject(output, vertexProperty.value());
- }
- }
- }
- }
-
- /**
- * If the returned {@link StarGraph} is null, that means that the {@link GraphFilter} filtered the vertex.
- */
- @Override
- public StarGraph read(final Kryo kryo, final Input input, final Class<StarGraph> aClass) {
- final StarGraph starGraph = StarGraph.open();
- input.readByte(); // version field ignored for now - for future use with backward compatibility
- starGraph.edgeProperties = kryo.readObjectOrNull(input, HashMap.class);
- starGraph.metaProperties = kryo.readObjectOrNull(input, HashMap.class);
- starGraph.addVertex(T.id, kryo.readClassAndObject(input), T.label, kryo.readObject(input, String.class));
- readEdges(kryo, input, starGraph, Direction.IN);
- readEdges(kryo, input, starGraph, Direction.OUT);
- if (kryo.readObject(input, Boolean.class)) {
- final int numberOfUniqueKeys = kryo.readObject(input, Integer.class);
- for (int i = 0; i < numberOfUniqueKeys; i++) {
- final String vertexPropertyKey = kryo.readObject(input, String.class);
- final int numberOfVertexPropertiesWithKey = kryo.readObject(input, Integer.class);
- for (int j = 0; j < numberOfVertexPropertiesWithKey; j++) {
- final Object id = kryo.readClassAndObject(input);
- final Object value = kryo.readClassAndObject(input);
- starGraph.starVertex.property(VertexProperty.Cardinality.list, vertexPropertyKey, value, T.id, id);
- }
- }
- }
- return this.graphFilter.hasFilter() ? starGraph.applyGraphFilter(this.graphFilter).orElse(null) : starGraph;
- }
-
- private void writeEdges(final Kryo kryo, final Output output, final StarGraph starGraph, final Direction direction) {
- // only write edges if there are some AND if the user requested them to be serialized AND if they match
- // the direction being serialized by the format
- final Map<String, List<Edge>> starEdges = direction.equals(Direction.OUT) ? starGraph.starVertex.outEdges : starGraph.starVertex.inEdges;
- final boolean writeEdges = null != starEdges && edgeDirectionToSerialize != null
- && (edgeDirectionToSerialize == direction || edgeDirectionToSerialize == Direction.BOTH);
- kryo.writeObject(output, writeEdges);
- if (writeEdges) {
- kryo.writeObject(output, starEdges.size());
- for (final Map.Entry<String, List<Edge>> edges : starEdges.entrySet()) {
- kryo.writeObject(output, edges.getKey());
- kryo.writeObject(output, edges.getValue().size());
- for (final Edge edge : edges.getValue()) {
- kryo.writeClassAndObject(output, edge.id());
- kryo.writeClassAndObject(output, direction.equals(Direction.OUT) ? edge.inVertex().id() : edge.outVertex().id());
- }
- }
- }
- }
-
- private void readEdges(final Kryo kryo, final Input input, final StarGraph starGraph, final Direction direction) {
- if (kryo.readObject(input, Boolean.class)) {
- final int numberOfUniqueLabels = kryo.readObject(input, Integer.class);
- for (int i = 0; i < numberOfUniqueLabels; i++) {
- final String edgeLabel = kryo.readObject(input, String.class);
- final int numberOfEdgesWithLabel = kryo.readObject(input, Integer.class);
- for (int j = 0; j < numberOfEdgesWithLabel; j++) {
- final Object edgeId = kryo.readClassAndObject(input);
- final Object adjacentVertexId = kryo.readClassAndObject(input);
- if (this.graphFilter.checkEdgeLegality(direction, edgeLabel).positive()) {
- if (direction.equals(Direction.OUT))
- starGraph.starVertex.addOutEdge(edgeLabel, starGraph.addVertex(T.id, adjacentVertexId), T.id, edgeId);
- else
- starGraph.starVertex.addInEdge(edgeLabel, starGraph.addVertex(T.id, adjacentVertexId), T.id, edgeId);
- } else if (null != starGraph.edgeProperties) {
- starGraph.edgeProperties.remove(edgeId);
- }
- }
- }
- }
- }
}
[12/18] incubator-tinkerpop git commit: worked with @dalaro to fix a
bug in HadoopPoolShimService. Reverted my last work on VertexProgramHelper.
Posted by ok...@apache.org.
worked with @dalaro to fix a bug in HadoopPoolShimService. Reverted my last work on VertexProgramHelper.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/90e31599
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/90e31599
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/90e31599
Branch: refs/heads/master
Commit: 90e3159969363e6a3383ffc64d58c27f76384a55
Parents: 797364c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Jun 6 16:55:42 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Jun 6 16:55:42 2016 -0600
----------------------------------------------------------------------
.../computer/util/VertexProgramHelper.java | 33 +++++---------------
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 17 +++++-----
.../structure/io/HadoopPoolShimService.java | 2 +-
.../gremlin/hadoop/HadoopGraphProvider.java | 2 +-
.../spark/process/computer/SparkExecutor.java | 11 +++----
5 files changed, 23 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90e31599/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
index 2b3a0b2..bc67866 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramHelper.java
@@ -25,13 +25,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.EdgeVertexStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.util.Serializer;
-import org.apache.tinkerpop.shaded.kryo.io.Input;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
@@ -42,8 +37,6 @@ import java.util.Set;
*/
public final class VertexProgramHelper {
- private static final GryoPool GRYO_POOL = GryoPool.build().create();
-
private VertexProgramHelper() {
}
@@ -74,33 +67,21 @@ public final class VertexProgramHelper {
final String byteString = Arrays.toString(Serializer.serializeObject(object));
configuration.setProperty(key, byteString.substring(1, byteString.length() - 1));
} catch (final IOException e) {
- try {
- final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- GRYO_POOL.doWithWriter(kryo -> kryo.writeObject(outputStream, object));
- String byteString = Arrays.toString(outputStream.toByteArray());
- configuration.setProperty(key, byteString.substring(1, byteString.length() - 1));
- } catch (final Exception e1) {
- throw new IllegalArgumentException(e1.getMessage(), e1);
- }
+ throw new IllegalArgumentException(e.getMessage(), e);
}
}
public static <T> T deserialize(final Configuration configuration, final String key) {
- final String[] stringBytes = configuration.getString(key).split(",");
- byte[] bytes = new byte[stringBytes.length];
- for (int i = 0; i < stringBytes.length; i++) {
- bytes[i] = Byte.valueOf(stringBytes[i].trim());
- }
try {
+ final String[] stringBytes = configuration.getString(key).split(",");
+ byte[] bytes = new byte[stringBytes.length];
+ for (int i = 0; i < stringBytes.length; i++) {
+ bytes[i] = Byte.valueOf(stringBytes[i].trim());
+ }
return (T) Serializer.deserializeObject(bytes);
} catch (final IOException | ClassNotFoundException e) {
- try {
- return (T) GRYO_POOL.readWithKryo(kryo -> kryo.readClassAndObject(new Input(new ByteArrayInputStream(bytes))));
- } catch (final Exception e1) {
- throw new IllegalArgumentException(e1.getMessage(), e1);
- }
+ throw new IllegalArgumentException(e.getMessage(), e);
}
-
}
public static <S, E> Traversal.Admin<S, E> reverse(final Traversal.Admin<S, E> traversal) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90e31599/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
index 9184dd0..fd57a3c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/KryoShimServiceLoader.java
@@ -51,6 +51,7 @@ public class KryoShimServiceLoader {
public static void applyConfiguration(Configuration conf) {
KryoShimServiceLoader.conf = conf;
+ load(true);
}
/**
@@ -195,20 +196,20 @@ public class KryoShimServiceLoader {
if (0 == result) {
log.warn("Found two {} implementations with the same canonical classname: {}. " +
- "This may indicate a problem with the classpath/classloader such as " +
- "duplicate or conflicting copies of the file " +
- "META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService.",
- a.getClass().getCanonicalName());
+ "This may indicate a problem with the classpath/classloader such as " +
+ "duplicate or conflicting copies of the file " +
+ "META-INF/services/org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService.",
+ a.getClass().getCanonicalName());
} else {
String winner = 0 < result ? a.getClass().getCanonicalName() : b.getClass().getCanonicalName();
log.warn("{} implementations {} and {} are tied with priority value {}. " +
- "Preferring {} to the other because it has a lexicographically greater classname. " +
- "Consider setting the system property \"{}\" instead of relying on priority tie-breaking.",
- KryoShimService.class.getSimpleName(), a, b, ap, winner, SHIM_CLASS_SYSTEM_PROPERTY);
+ "Preferring {} to the other because it has a lexicographically greater classname. " +
+ "Consider setting the system property \"{}\" instead of relying on priority tie-breaking.",
+ KryoShimService.class.getSimpleName(), a, b, ap, winner, SHIM_CLASS_SYSTEM_PROPERTY);
}
return result;
}
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90e31599/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
index 5753d90..df72b71 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/HadoopPoolShimService.java
@@ -71,6 +71,6 @@ public class HadoopPoolShimService implements KryoShimService {
@Override
public void applyConfiguration(Configuration conf) {
- KryoShimServiceLoader.applyConfiguration(conf);
+ HadoopPools.initialize(conf);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90e31599/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index 57157db..e36c08d 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -112,7 +112,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- System.setProperty(SHIM_CLASS_SYSTEM_PROPERTY, HadoopPoolShimService.class.getCanonicalName());
+ System.clearProperty(SHIM_CLASS_SYSTEM_PROPERTY);
this.graphSONInput = RANDOM.nextBoolean();
return new HashMap<String, Object>() {{
put(Graph.GRAPH, HadoopGraph.class.getName());
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90e31599/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index 9e5ac53..4db8086 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -24,7 +24,6 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
@@ -171,7 +170,7 @@ public final class SparkExecutor {
assert graphRDD.partitioner().get().equals(newViewIncomingRDD.partitioner().get());
newViewIncomingRDD
.foreachPartition(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
}); // need to complete a task so its BSP and the memory for this iteration is updated
return newViewIncomingRDD;
}
@@ -206,7 +205,7 @@ public final class SparkExecutor {
final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce,
final Configuration apacheConfiguration) {
JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
return () -> new MapIterator<>(MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration), partitionIterator);
});
if (mapReduce.getMapKeySort().isPresent())
@@ -217,7 +216,7 @@ public final class SparkExecutor {
public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeCombine(final JavaPairRDD<K, V> mapRDD,
final Configuration apacheConfiguration) {
return mapRDD.mapPartitionsToPair(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
return () -> new CombineIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration), partitionIterator);
});
}
@@ -226,11 +225,11 @@ public final class SparkExecutor {
final JavaPairRDD<K, V> mapOrCombineRDD, final MapReduce<K, V, OK, OV, ?> mapReduce,
final Configuration apacheConfiguration) {
JavaPairRDD<OK, OV> reduceRDD = mapOrCombineRDD.groupByKey().mapPartitionsToPair(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
+ KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
return () -> new ReduceIterator<>(MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration), partitionIterator);
});
if (mapReduce.getReduceKeySort().isPresent())
reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get(), true, 1);
return reduceRDD;
}
-}
+}
\ No newline at end of file
[09/18] incubator-tinkerpop git commit: Merge branch
'serialization-flexibility' of https://github.com/dalaro/incubator-tinkerpop
into TINKERPOP-1321-dan
Posted by ok...@apache.org.
Merge branch 'serialization-flexibility' of https://github.com/dalaro/incubator-tinkerpop into TINKERPOP-1321-dan
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/0cd31bf8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/0cd31bf8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/0cd31bf8
Branch: refs/heads/master
Commit: 0cd31bf8e86a26ac92f232fdcb6a29eda9d6dfd9
Parents: 55eb1e6 88ac304
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon Jun 6 13:36:54 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon Jun 6 13:36:54 2016 -0600
----------------------------------------------------------------------
.../process/computer/GiraphWorkerContext.java | 3 +-
.../gremlin/structure/io/gryo/GryoMapper.java | 440 ++++++++++++-------
.../gremlin/structure/io/gryo/GryoPool.java | 1 +
.../structure/io/gryo/GryoSerializers.java | 40 +-
.../structure/io/gryo/JavaTimeSerializers.java | 125 ++----
.../structure/io/gryo/PairSerializer.java | 11 +-
.../structure/io/gryo/TypeRegistration.java | 82 ++++
.../structure/io/gryo/URISerializer.java | 23 +-
.../structure/io/gryo/UUIDSerializer.java | 24 +-
.../structure/io/gryo/kryoshim/InputShim.java | 38 ++
.../structure/io/gryo/kryoshim/KryoShim.java | 41 ++
.../io/gryo/kryoshim/KryoShimService.java | 99 +++++
.../io/gryo/kryoshim/KryoShimServiceLoader.java | 214 +++++++++
.../structure/io/gryo/kryoshim/OutputShim.java | 40 ++
.../io/gryo/kryoshim/SerializerShim.java | 36 ++
.../io/gryo/kryoshim/package-info.java | 55 +++
.../kryoshim/shaded/ShadedInputAdapter.java | 66 +++
.../gryo/kryoshim/shaded/ShadedKryoAdapter.java | 67 +++
.../kryoshim/shaded/ShadedOutputAdapter.java | 72 +++
.../shaded/ShadedSerializerAdapter.java | 54 +++
.../io/gryo/kryoshim/shaded/package-info.java | 25 ++
.../util/star/StarGraphGryoSerializer.java | 130 +-----
.../util/star/StarGraphSerializer.java | 150 +++++++
.../hadoop/process/computer/HadoopCombine.java | 3 +-
.../hadoop/process/computer/HadoopMap.java | 3 +-
.../hadoop/process/computer/HadoopReduce.java | 3 +-
.../structure/io/HadoopPoolShimService.java | 76 ++++
.../hadoop/structure/io/HadoopPools.java | 1 +
.../structure/io/HadoopPoolsConfigurable.java | 4 +-
.../hadoop/structure/io/ObjectWritable.java | 25 +-
.../hadoop/structure/io/VertexWritable.java | 39 +-
.../structure/io/gryo/GryoRecordReader.java | 3 +-
.../structure/io/gryo/GryoRecordWriter.java | 4 +-
...n.structure.io.gryo.kryoshim.KryoShimService | 1 +
.../spark/process/computer/SparkExecutor.java | 3 +-
.../structure/io/gryo/GryoRegistrator.java | 194 ++++++++
.../spark/structure/io/gryo/GryoSerializer.java | 2 +-
.../io/gryo/IoRegistryAwareKryoSerializer.java | 116 +++++
.../io/gryo/ObjectWritableSerializer.java | 12 +-
.../io/gryo/VertexWritableSerializer.java | 12 +-
.../kryoshim/unshaded/UnshadedInputAdapter.java | 78 ++++
.../kryoshim/unshaded/UnshadedKryoAdapter.java | 74 ++++
.../unshaded/UnshadedKryoShimService.java | 154 +++++++
.../unshaded/UnshadedOutputAdapter.java | 83 ++++
.../unshaded/UnshadedSerializerAdapter.java | 57 +++
...n.structure.io.gryo.kryoshim.KryoShimService | 1 +
.../computer/SparkHadoopGraphProvider.java | 11 +-
.../spark/structure/io/ToyGraphInputRDD.java | 3 +-
48 files changed, 2328 insertions(+), 470 deletions(-)
----------------------------------------------------------------------
[05/18] incubator-tinkerpop git commit: Kryo shim configuration tweaks
Posted by ok...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/9321a3e1/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
index ea3636f..4cd8cea 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -30,6 +30,7 @@ import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
@@ -46,7 +47,7 @@ public final class ToyGraphInputRDD implements InputRDD {
@Override
public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- HadoopPools.initialize(TinkerGraph.open().configuration());
+ KryoShimServiceLoader.applyConfiguration(TinkerGraph.open().configuration());
final List<VertexWritable> vertices;
if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("modern"))
vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new));