You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2018/08/01 19:29:37 UTC
[13/50] [abbrv] tinkerpop git commit: TINKERPOP-1996 Added support
for setting IoRegistries using with()
TINKERPOP-1996 Added support for setting IoRegistries using with()
IORegistry instances are important because they feed serializer information to the Reader/Writer instances. Of all the configuration options that one seemed like the most important to make possible using with().
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ff71c6ab
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ff71c6ab
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ff71c6ab
Branch: refs/heads/TINKERPOP-1990
Commit: ff71c6abee0b39d7ee95128c3d64906daad96a76
Parents: ae3b149
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Thu Jul 19 16:13:57 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 16:13:57 2018 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/process/traversal/IO.java | 7 ++
.../traversal/step/sideEffect/IoStep.java | 75 ++++++++++++--------
.../process/traversal/step/util/Parameters.java | 4 +-
.../Process/Traversal/GraphTraversal.cs | 21 ++----
.../src/Gremlin.Net/Process/Traversal/IO.cs | 2 +
.../gremlin-javascript/lib/process/traversal.js | 4 ++
.../jython/gremlin_python/process/traversal.py | 2 +
.../gremlin/structure/io/util/CustomId.java | 39 ++++++++++
.../step/sideEffect/TinkerGraphIoStepTest.java | 75 ++++++++++++++++++++
9 files changed, 181 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
index 6668cf1..67b4670 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
@@ -73,4 +74,10 @@ public class IO {
* the file extension provided to it.
*/
public static final String writer = Graph.Hidden.hide("tinkerpop.io.writer");
+
+ /**
+ * A key that identifies the fully qualified class names of {@link IoRegistry} instances to use. May be specified
+ * multiple times (i.e. once for each registry) using the {@link GraphTraversal#with(String, Object)} modulator.
+ */
+ public static final String registry = Graph.Hidden.hide("tinkerpop.io.registry");
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
index 9804333..1d4f40b 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java
@@ -31,10 +31,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementExce
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
@@ -46,6 +49,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
/**
* Handles read and write operations into the {@link Graph}.
@@ -138,15 +144,19 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
* extension or simply uses configurations provided by the user on the parameters given to the step.
*/
private GraphReader constructReader() {
- final Object objectOrClass = parameters.get(IO.reader, this::detectReader).get(0);
+ final Object objectOrClass = parameters.get(IO.reader, this::detectFileType).get(0);
if (objectOrClass instanceof GraphReader)
return (GraphReader) objectOrClass;
else if (objectOrClass instanceof String) {
- if (objectOrClass.equals(IO.graphson))
- return GraphSONReader.build().create();
- else if (objectOrClass.equals(IO.gryo))
- return GryoReader.build().create();
- else if (objectOrClass.equals(IO.graphml))
+ if (objectOrClass.equals(IO.graphson)) {
+ final GraphSONMapper.Builder builder = GraphSONMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GraphSONReader.build().mapper(builder.create()).create();
+ } else if (objectOrClass.equals(IO.gryo)){
+ final GryoMapper.Builder builder = GryoMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GryoReader.build().mapper(builder.create()).create();
+ } else if (objectOrClass.equals(IO.graphml))
return GraphMLReader.build().create();
else {
try {
@@ -163,31 +173,24 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
}
}
- private GraphReader detectReader() {
- if (file.endsWith(".kryo"))
- return GryoReader.build().create();
- else if (file.endsWith(".json"))
- return GraphSONReader.build().create();
- else if (file.endsWith(".xml"))
- return GraphMLReader.build().create();
- else
- throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
- }
-
/**
* Builds a {@link GraphWriter} instance to use. Attempts to detect the file format to be write using the file
* extension or simply uses configurations provided by the user on the parameters given to the step.
*/
private GraphWriter constructWriter() {
- final Object objectOrClass = parameters.get(IO.writer, this::detectWriter).get(0);
+ final Object objectOrClass = parameters.get(IO.writer, this::detectFileType).get(0);
if (objectOrClass instanceof GraphWriter)
return (GraphWriter) objectOrClass;
else if (objectOrClass instanceof String) {
- if (objectOrClass.equals(IO.graphson))
- return GraphSONWriter.build().create();
- else if (objectOrClass.equals(IO.gryo))
- return GryoWriter.build().create();
- else if (objectOrClass.equals(IO.graphml))
+ if (objectOrClass.equals(IO.graphson)) {
+ final GraphSONMapper.Builder builder = GraphSONMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GraphSONWriter.build().mapper(builder.create()).create();
+ } else if (objectOrClass.equals(IO.gryo)){
+ final GryoMapper.Builder builder = GryoMapper.build();
+ detectRegistries().forEach(builder::addRegistry);
+ return GryoWriter.build().mapper(builder.create()).create();
+ }else if (objectOrClass.equals(IO.graphml))
return GraphMLWriter.build().create();
else {
try {
@@ -204,21 +207,31 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting {
}
}
- private GraphWriter detectWriter() {
+ private String detectFileType() {
if (file.endsWith(".kryo"))
- return GryoWriter.build().create();
+ return IO.gryo;
else if (file.endsWith(".json"))
- return GraphSONWriter.build().create();
+ return IO.graphson;
else if (file.endsWith(".xml"))
- return GraphMLWriter.build().create();
+ return IO.graphml;
else
throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension");
}
- private Configuration getConfFromParameters() {
- final Configuration conf = new BaseConfiguration();
- parameters.getRaw().forEach((key, value) -> conf.setProperty(key.toString(), value.get(0)));
- return conf;
+ private List<IoRegistry> detectRegistries() {
+ final List<Object> k = parameters.get(IO.registry, Collections::emptyList);
+ return parameters.get(IO.registry, null).stream().map(cn -> {
+ try {
+ if (cn instanceof IoRegistry)
+ return (IoRegistry) cn;
+ else {
+ final Class<?> clazz = Class.forName(cn.toString());
+ return (IoRegistry) clazz.getMethod("instance").invoke(null);
+ }
+ } catch (Exception ex) {
+ throw new IllegalStateException(ex);
+ }
+ }).collect(Collectors.toList());
}
@Override
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
index 40d9330..eb57f4b 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java
@@ -100,11 +100,11 @@ public final class Parameters implements Cloneable, Serializable {
* Gets the value of a key and if that key isn't present returns the default value from the {@link Supplier}.
*
* @param key the key to retrieve
- * @param defaultValue the default value generator
+ * @param defaultValue the default value generator which if null will return an empty list
*/
public <E> List<E> get(final Object key, final Supplier<E> defaultValue) {
final List<E> list = (List<E>) this.parameters.get(key);
- return (null == list) ? Collections.singletonList(defaultValue.get()) : list;
+ return (null == list) ? (null == defaultValue ? Collections.emptyList() : Collections.singletonList(defaultValue.get())) : list;
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
index 82d72c0..361b246 100644
--- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs
@@ -1256,20 +1256,20 @@ namespace Gremlin.Net.Process.Traversal
}
/// <summary>
- /// Adds the repeat step to this <see cref="GraphTraversal{SType, EType}" />.
+ /// Adds the read step to this <see cref="GraphTraversal{SType, EType}" />.
/// </summary>
- public GraphTraversal<S, E> Repeat (string loopName, ITraversal repeatTraversal)
+ public GraphTraversal<S, E> Read ()
{
- Bytecode.AddStep("repeat", loopName, repeatTraversal);
+ Bytecode.AddStep("read");
return Wrap<S, E>(this);
}
/// <summary>
- /// Adds the read step to this <see cref="GraphTraversal{SType, EType}" />.
+ /// Adds the repeat step to this <see cref="GraphTraversal{SType, EType}" />.
/// </summary>
- public GraphTraversal<S, E> Read ()
+ public GraphTraversal<S, E> Repeat (string loopName, ITraversal repeatTraversal)
{
- Bytecode.AddStep("read");
+ Bytecode.AddStep("repeat", loopName, repeatTraversal);
return Wrap<S, E>(this);
}
@@ -1713,15 +1713,6 @@ namespace Gremlin.Net.Process.Traversal
}
/// <summary>
- /// Adds the with step to this <see cref="GraphTraversal{SType, EType}" />.
- /// </summary>
- public GraphTraversal<S, E> With (string key, object value)
- {
- Bytecode.AddStep("with", key, value);
- return Wrap<S, E>(this);
- }
-
- /// <summary>
/// Adds the write step to this <see cref="GraphTraversal{SType, EType}" />.
/// </summary>
public GraphTraversal<S, E> Write ()
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
----------------------------------------------------------------------
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
index 288f7e3..861b431 100644
--- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs
@@ -46,6 +46,8 @@ namespace Gremlin.Net.Process.Traversal
public const String reader = "~tinkerpop.io.reader";
+ public const String registry = "~tinkerpop.io.registry";
+
public const String writer = "~tinkerpop.io.writer";
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
----------------------------------------------------------------------
diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
index 3f69fb1..09aec91 100644
--- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
+++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js
@@ -129,6 +129,10 @@ class IO {
return "~tinkerpop.io.reader"
}
+ static get registry() {
+ return "~tinkerpop.io.registry"
+ }
+
static get writer() {
return "~tinkerpop.io.writer"
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
----------------------------------------------------------------------
diff --git a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
index d9fb4d9..49bb7b1 100644
--- a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
+++ b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py
@@ -319,6 +319,8 @@ class IO(object):
reader = "~tinkerpop.io.reader"
+ registry = "~tinkerpop.io.registry"
+
writer = "~tinkerpop.io.writer"
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
index d503ae8..0ab3b90 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java
@@ -18,9 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.structure.io.util;
+import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.Io;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.graphson.AbstractObjectDeserializer;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens;
import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException;
import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
import org.apache.tinkerpop.shaded.jackson.core.JsonParser;
@@ -31,11 +36,13 @@ import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer;
import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer;
import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer;
+import org.javatuples.Pair;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -66,6 +73,24 @@ public class CustomId {
}
@Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ final CustomId customId = (CustomId) o;
+
+ if (!cluster.equals(customId.cluster)) return false;
+ return elementId.equals(customId.elementId);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = cluster.hashCode();
+ result = 31 * result + elementId.hashCode();
+ return result;
+ }
+
+ @Override
public String toString() {
return cluster + ":" + elementId;
}
@@ -219,4 +244,18 @@ public class CustomId {
return "simple";
}
}
+
+ public static class CustomIdIoRegistry extends AbstractIoRegistry {
+
+ private static final CustomIdIoRegistry INSTANCE = new CustomIdIoRegistry();
+
+ private CustomIdIoRegistry() {
+ register(GryoIo.class, CustomId.class, null);
+ register(GraphSONIo.class, null, new CustomIdTinkerPopJacksonModuleV3d0());
+ }
+
+ public static CustomIdIoRegistry instance() {
+ return INSTANCE;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
new file mode 100644
index 0000000..06c4db8
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.process.traversal.step.sideEffect;
+
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.process.traversal.IO;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.util.CustomId;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.UUID;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * It was hard to test the {@link IO#registry} configuration as a generic test. Opted to test it as a bit of a
+ * standalone test with TinkerGraph.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public class TinkerGraphIoStepTest {
+
+ private Graph graph;
+ private GraphTraversalSource g;
+
+ @Before
+ public void setup() {
+ graph = TinkerGraph.open();
+ g = graph.traversal();
+ }
+
+ @Test
+ public void shouldWriteReadWithCustomIoRegistryGryo() throws Exception {
+ final UUID uuid = UUID.randomUUID();
+ g.addV("person").property("name","stephen").property("custom", new CustomId("a", uuid)).iterate();
+
+ final File file = TestHelper.generateTempFile(TinkerGraphIoStepTest.class, "shouldWriteReadWithCustomIoRegistryGryo", ".kryo");
+ g.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.class.getName()).write().iterate();
+
+ final Graph emptyGraph = TinkerGraph.open();
+ final GraphTraversalSource emptyG = emptyGraph.traversal();
+
+ try {
+ emptyG.io(file.getAbsolutePath()).read().iterate();
+ fail("Can't read without a registry");
+ } catch (Exception ignored) {
+ // do nothing
+ }
+
+ emptyG.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.instance()).read().iterate();
+
+ assertEquals(1, emptyG.V().has("custom", new CustomId("a", uuid)).count().next().intValue());
+ }
+}