You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/04/21 18:18:03 UTC
[09/12] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master' into refactor-io
Merge remote-tracking branch 'origin/master' into refactor-io
Conflicts:
gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/6ffd5b4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/6ffd5b4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/6ffd5b4c
Branch: refs/heads/master
Commit: 6ffd5b4ce740a5be1e8cf4eacaac66a8420e0981
Parents: 42f367a 19f83de
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Apr 17 12:33:44 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Apr 17 12:33:44 2015 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 2 +
README.asciidoc | 1 +
data/grateful-dead-vertices.kryo | Bin 1028298 -> 975214 bytes
data/grateful-dead-vertices.ldjson | 422 +++++++++----------
data/grateful-dead.kryo | Bin 515409 -> 493339 bytes
data/tinkerpop-classic-typed.json | 2 +-
data/tinkerpop-classic-vertices.kryo | Bin 1323 -> 1179 bytes
data/tinkerpop-classic-vertices.ldjson | 12 +-
data/tinkerpop-classic.json | 2 +-
data/tinkerpop-classic.kryo | Bin 817 -> 765 bytes
data/tinkerpop-crew-vertices.kryo | Bin 2743 -> 2575 bytes
data/tinkerpop-crew.kryo | Bin 1916 -> 1858 bytes
data/tinkerpop-modern-vertices.kryo | Bin 1439 -> 1295 bytes
data/tinkerpop-modern.kryo | Bin 877 -> 825 bytes
.../gremlin/structure/io/gryo/GryoMapper.java | 212 +---------
.../gremlin/structure/io/gryo/GryoReader.java | 18 +-
.../gremlin/structure/io/gryo/GryoWriter.java | 25 +-
.../structure/io/gryo/GryoReaderWriterTest.java | 102 +++++
.../driver/ser/GryoMessageSerializerV1d0.java | 19 +-
.../structure/io/graphml/grateful-dead.xml | 18 +-
.../structure/io/graphml/tinkerpop-classic.xml | 18 +-
.../structure/io/graphml/tinkerpop-modern.xml | 18 +-
.../graphson/tinkerpop-classic-normalized.json | 2 +-
.../io/graphson/tinkerpop-classic-typed.json | 2 +-
.../io/graphson/tinkerpop-classic.json | 2 +-
.../io/gryo/grateful-dead-vertices.kryo | Bin 1028298 -> 975214 bytes
.../structure/io/gryo/grateful-dead.kryo | Bin 515409 -> 493339 bytes
.../io/gryo/tinkerpop-classic-vertices.kryo | Bin 1323 -> 1179 bytes
.../structure/io/gryo/tinkerpop-classic.kryo | Bin 817 -> 765 bytes
.../io/gryo/tinkerpop-crew-vertices.kryo | Bin 2743 -> 2575 bytes
.../structure/io/gryo/tinkerpop-crew.kryo | Bin 1916 -> 1858 bytes
.../io/gryo/tinkerpop-modern-vertices.kryo | Bin 1439 -> 1295 bytes
.../structure/io/gryo/tinkerpop-modern.kryo | Bin 877 -> 825 bytes
.../structure/io/gryo/GryoRecordReader.java | 8 +-
tinkergraph-gremlin/pom.xml | 2 +-
.../tinkergraph/structure/TinkerGraphTest.java | 4 +-
36 files changed, 384 insertions(+), 507 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6ffd5b4c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --cc gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index f50198f,9bdf9d4..f6ee967
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@@ -53,14 -52,10 +53,11 @@@ import org.apache.tinkerpop.gremlin.str
import org.apache.tinkerpop.shaded.kryo.Kryo;
import org.apache.tinkerpop.shaded.kryo.KryoSerializable;
import org.apache.tinkerpop.shaded.kryo.Serializer;
- import org.apache.tinkerpop.shaded.kryo.io.Input;
- import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.apache.tinkerpop.shaded.kryo.util.DefaultStreamFactory;
import org.apache.tinkerpop.shaded.kryo.util.MapReferenceResolver;
+import org.javatuples.Pair;
import org.javatuples.Triplet;
- import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.URI;
@@@ -173,69 -121,9 +123,9 @@@ public final class GryoMapper implement
}
/**
- * Use the most current version of Gryo.
+ * A builder to construct a {@link GryoMapper} instance.
*/
- public static Builder build() {
- return Version.V_1_0_0.getBuilder();
- }
-
- public static interface Builder extends Mapper.Builder<Builder> {
- /**
- * Add mapper classes to serializes with gryo using standard serialization.
- */
- public Builder addCustom(final Class... custom);
-
- /**
- * Add mapper class to serializes with mapper serialization.
- */
- public Builder addCustom(final Class clazz, final Serializer serializer);
-
- /**
- * Add mapper class to serializes with mapper serialization as returned from a {@link Function}.
- */
- public Builder addCustom(final Class clazz, final Function<Kryo, Serializer> serializer);
-
- /**
- * If using mapper classes it might be useful to tag the version stamped to the serialization with a mapper
- * value, such that Gryo serialization at 1.0.0 would have a fourth byte for an extended version. The user
- * supplied fourth byte can then be used to ensure the right deserializer is used to read the data. If this
- * value is not supplied then it is written as {@link Byte#MIN_VALUE}. The value supplied here should be greater
- * than or equal to zero.
- */
- public Builder extendedVersion(final byte extendedVersion);
-
- /**
- * By default the {@link #extendedVersion(byte)} is checked against what is read from an input source and if
- * those values are equal the version being read is considered "compliant". To alter this behavior, supply a
- * mapper compliance {@link Predicate} to evaluate the value read from the input source (i.e. first argument)
- * and the value marked in the {@code GryoMapper} instance {i.e. second argument}. Supplying this function is
- * useful when versions require backward compatibility or other more complex checks. This function is only used
- * if the {@link #extendedVersion(byte)} is set to something other than its default.
- */
- public Builder compliant(final BiPredicate<Byte, Byte> compliant);
-
- public GryoMapper create();
- }
-
- public enum Version {
- V_1_0_0(BuilderV1d0.class);
-
- private final Class<? extends Builder> builder;
-
- private Version(final Class<? extends Builder> builder) {
- this.builder = builder;
- }
-
- Builder getBuilder() {
- try {
- return builder.newInstance();
- } catch (Exception x) {
- throw new RuntimeException("Gryo Builder implementation cannot be instantiated", x);
- }
- }
- }
-
- public static class BuilderV1d0 implements Builder {
- public static class Builder {
++ public static class Builder implements Mapper.Builder<Builder> {
/**
* Map with one entry that is used so that it is possible to get the class of LinkedHashMap.Entry.
@@@ -330,30 -218,16 +220,24 @@@
add(Triplet.<Class, Function<Kryo, Serializer>, Integer>with(DependantMutableMetrics.class, null, 80));
}};
- private static final byte major = 1;
- private static final byte minor = 0;
- private static final byte patchLevel = 0;
-
- private byte extendedVersion = DEFAULT_EXTENDED_VERSION;
- private BiPredicate<Byte, Byte> compliant = (readExt, serExt) -> readExt.equals(serExt);
-
+ private IoRegistry registry = null;
+
/**
* Starts numbering classes for Gryo serialization at 65536 to leave room for future usage by TinkerPop.
*/
private final AtomicInteger currentSerializationId = new AtomicInteger(65536);
--
++
+ private Builder() {}
++
+ @Override
+ public Builder addRegistry(final IoRegistry registry) {
+ this.registry = registry;
+ return this;
+ }
/**
- * {@inheritDoc}
- */
- @Override
+ * Register custom classes to serializes with gryo using default serialization.
- */
++ */
public Builder addCustom(final Class... custom) {
if (custom != null && custom.length > 0)
serializationList.addAll(Arrays.asList(custom).stream()
@@@ -372,92 -245,15 +255,29 @@@
}
/**
- * {@inheritDoc}
- */
- @Override
+ * Register a custom class to serialize with a custom serializer as returned from a {@link Function}.
- */
++ */
public Builder addCustom(final Class clazz, final Function<Kryo, Serializer> serializer) {
serializationList.add(Triplet.with(clazz, serializer, currentSerializationId.getAndIncrement()));
return this;
}
- /**
- * {@inheritDoc}
- */
- @Override
- public Builder extendedVersion(final byte extendedVersion) {
- if (extendedVersion > DEFAULT_EXTENDED_VERSION && extendedVersion < 0)
- throw new IllegalArgumentException("The extendedVersion must be greater than zero");
-
- this.extendedVersion = extendedVersion;
- return this;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Builder compliant(final BiPredicate<Byte, Byte> compliant) {
- if (null == compliant)
- throw new IllegalArgumentException("compliant");
-
- this.compliant = compliant;
- return this;
- }
-
- @Override
public GryoMapper create() {
+ if (registry != null) {
+ final List<Pair<Class, Object>> serializers = registry.find(GryoIo.class);
+ serializers.forEach(p -> {
+ if (null == p.getValue1())
+ addCustom(p.getValue0());
+ else if (p.getValue1() instanceof Serializer)
+ addCustom(p.getValue0(), (Serializer) p.getValue1());
+ else if (p.getValue1() instanceof Function)
+ addCustom(p.getValue0(), (Function<Kryo, Serializer>) p.getValue1());
+ else
+ throw new RuntimeException("Invalid serializer"); // todo: cleanup exception handling
+ });
+ }
+
- return new GryoMapper(serializationList, this::writeHeader, this::readHeader);
- }
-
- private void writeHeader(final Kryo kryo, final Output output) throws IOException {
- // 32 byte header total
- output.writeBytes(GIO);
-
- // some space for later
- output.writeBytes(new byte[25]);
-
- // version x.y.z
- output.writeByte(major);
- output.writeByte(minor);
- output.writeByte(patchLevel);
- output.writeByte(extendedVersion);
- }
-
- private void readHeader(final Kryo kryo, final Input input) throws IOException {
- if (!Arrays.equals(GIO, input.readBytes(3)))
- throw new IOException("Invalid format - first three bytes of header do not match expected value");
-
- // skip the next 25 bytes in v1
- input.readBytes(25);
-
- // final three bytes of header are the version which should be 1.0.0
- final byte[] version = input.readBytes(3);
- final byte extension = input.readByte();
-
- // direct match on version for now
- if (version[0] != major || version[1] != minor || version[2] != patchLevel)
- throw new IOException(String.format(
- "The version [%s.%s.%s] in the stream cannot be understood by this reader",
- version[0], version[1], version[2]));
-
- if (extendedVersion >= 0 && !compliant.test(extension, extendedVersion))
- throw new IOException(String.format(
- "The extension [%s] in the input source is not compliant with this configuration of Gryo - [%s]",
- extension, extendedVersion));
+ return new GryoMapper(serializationList);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6ffd5b4c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6ffd5b4c/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
----------------------------------------------------------------------
diff --cc gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
index b7c82a7,3b011ef..25204a9
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoWriter.java
@@@ -30,11 -30,8 +30,10 @@@ import org.apache.tinkerpop.shaded.kryo
import java.io.IOException;
import java.io.OutputStream;
+import java.io.Writer;
+import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Iterator;
- import java.util.UUID;
/**
* The {@link GraphWriter} for the Gremlin Structure serialization format based on Kryo. The format is meant to be
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6ffd5b4c/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
----------------------------------------------------------------------
diff --cc gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
index ff50d6a,9d44eae..8586bda
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/ser/GryoMessageSerializerV1d0.java
@@@ -110,7 -101,7 +102,7 @@@ public class GryoMessageSerializerV1d0
// a graph was found so use the mapper it constructs. this allows gryo to be auto-configured with any
// custom classes that the implementation allows for
- initialBuilder = g.io(GryoIo.build()).mapper();
- builder = g.io().gryoMapper();
++ builder = g.io(GryoIo.build()).mapper();
} else {
// no graph was supplied so just use the default - this will likely be the case when using a graph
// with no custom classes or a situation where the user needs complete control like when using two
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/6ffd5b4c/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphTest.java
----------------------------------------------------------------------