You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2018/08/01 19:29:31 UTC
[07/50] [abbrv] tinkerpop git commit: TINKERPOP-1996 Got read/write()
tests running for OLAP
TINKERPOP-1996 Got read/write() tests running for OLAP
Introduced new Graph.Features to provider better separation between graph mutation features and graph loading features - they are two different things as demonstrated by io(). Fixed HadoopIoStep/Strategy so that they properly handle the different input/output format types expected.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/f148e933
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/f148e933
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/f148e933
Branch: refs/heads/TINKERPOP-1990
Commit: f148e9331a945e0f4f707ea937b610e5902701c7
Parents: bd275a7
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Jul 13 15:17:17 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Thu Jul 19 13:40:10 2018 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/structure/Graph.java | 27 +++++++++
.../gremlin/AbstractGraphProvider.java | 18 ++++--
.../gremlin/process/ProcessComputerSuite.java | 4 ++
.../process/traversal/step/map/ReadTest.java | 21 ++++---
.../process/traversal/step/map/WriteTest.java | 14 ++++-
.../traversal/step/map/HadoopIoStep.java | 60 +++++++++++++++++++-
.../traversal/strategy/HadoopIoStrategy.java | 2 +-
.../gremlin/hadoop/structure/HadoopGraph.java | 16 ++++++
8 files changed, 142 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
index f1fc54a..f62b897 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java
@@ -23,6 +23,8 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.util.FeatureDescriptor;
@@ -439,6 +441,8 @@ public interface Graph extends AutoCloseable, Host {
public static final String FEATURE_PERSISTENCE = "Persistence";
public static final String FEATURE_THREADED_TRANSACTIONS = "ThreadedTransactions";
public static final String FEATURE_CONCURRENT_ACCESS = "ConcurrentAccess";
+ public static final String FEATURE_IO_READ = "IoRead";
+ public static final String FEATURE_IO_WRITE = "IoWrite";
/**
* Determines if the {@code Graph} implementation supports {@link GraphComputer} based processing.
@@ -489,6 +493,29 @@ public interface Graph extends AutoCloseable, Host {
}
/**
+ * Determines if the {@code Graph} implementations supports read operations as executed with the
+ * {@link GraphTraversalSource#io(String)} step. Graph implementations will generally support this by
+ * default as any graph that can support direct mutation through the Structure API will by default
+ * accept data from the standard TinkerPop {@link GraphReader} implementations. However, some graphs like
+ * {@code HadoopGraph} don't accept direct mutations but can still do reads from that {@code io()} step.
+ */
+ @FeatureDescriptor(name = FEATURE_IO_READ)
+ public default boolean supportsIoRead() {
+ return true;
+ }
+
+ /**
+ * Determines if the {@code Graph} implementations supports write operations as executed with the
+ * {@link GraphTraversalSource#io(String)} step. Graph implementations will generally support this by
+ * default given the standard TinkerPop {@link GraphWriter} implementations. However, some graphs like
+ * {@code HadoopGraph} will use a different approach to handle writes.
+ */
+ @FeatureDescriptor(name = FEATURE_IO_WRITE)
+ public default boolean supportsIoWrite() {
+ return true;
+ }
+
+ /**
* Gets the features related to "graph sideEffects" operation.
*/
public default VariableFeatures variables() {
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
index 95c6b57..75d033b 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java
@@ -20,6 +20,7 @@ package org.apache.tinkerpop.gremlin;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.commons.configuration.BaseConfiguration;
@@ -131,12 +132,19 @@ public abstract class AbstractGraphProvider implements GraphProvider {
return methodName.replaceAll("[0-9, -]+$", String.valueOf(random));
}
- protected void readIntoGraph(final Graph g, final String path) throws IOException {
- final GraphReader reader = GryoReader.build()
- .mapper(g.io(GryoIo.build()).mapper().create())
- .create();
+ /**
+ * Used by the default implementation of {@link AbstractGraphProvider#loadGraphData(Graph, LoadGraphWith, Class, String)}
+ * to read the graph from a Kryo file using the default {@link GryoReader} implementation. If the default
+ * implementation does not work (perhaps a graph implementation needs to register some special {@link IoRegistry}
+ * then this method or its caller should be overridden to suit the implementation.
+ *
+ * @param graph the graph to load to
+ * @param path the path to the file to load into the graph
+ */
+ protected void readIntoGraph(final Graph graph, final String path) throws IOException {
+ final GraphReader reader = GryoReader.build().create();
try (final InputStream stream = AbstractGremlinTest.class.getResourceAsStream(path)) {
- reader.readGraph(stream, g);
+ reader.readGraph(stream, graph);
}
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
index b224c8b..eab562d 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
@@ -70,11 +70,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ExplainTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest;
@@ -163,10 +165,12 @@ public class ProcessComputerSuite extends AbstractGremlinSuite {
ProjectTest.Traversals.class,
ProgramTest.Traversals.class,
PropertiesTest.Traversals.class,
+ ReadTest.Traversals.class,
SelectTest.Traversals.class,
UnfoldTest.Traversals.class,
ValueMapTest.Traversals.class,
VertexTest.Traversals.class,
+ WriteTest.Traversals.class,
// sideEffect
AddEdgeTest.Traversals.class,
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
index 9e53169..ab59194 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java
@@ -18,12 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+import org.apache.tinkerpop.gremlin.FeatureRequirement;
import org.apache.tinkerpop.gremlin.FeatureRequirementSet;
import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.IoTest;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLResourceAccess;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
@@ -33,6 +35,7 @@ import org.junit.runner.RunWith;
import java.io.IOException;
+import static org.apache.tinkerpop.gremlin.structure.Graph.Features.GraphFeatures.FEATURE_IO_READ;
import static org.junit.Assert.assertFalse;
/**
@@ -54,8 +57,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
public abstract Traversal<Object,Object> get_g_io_read_withXreader_graphmlX(final String fileToRead) throws IOException;
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
- public void g_readXkryoX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
+ public void g_io_readXkryoX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXkryoX(fileToRead);
printTraversalForm(traversal);
@@ -65,7 +68,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
public void g_io_read_withXreader_gryoX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_gryoX(fileToRead);
@@ -76,8 +79,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
- public void g_readXjsonX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
+ public void g_io_readXjsonX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXjsonX(fileToRead);
printTraversalForm(traversal);
@@ -87,7 +90,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
public void g_io_read_withXreader_graphsonX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphsonX(fileToRead);
@@ -98,8 +101,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
- public void g_readXxmlX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
+ public void g_io_readXxmlX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_readXxmlX(fileToRead);
printTraversalForm(traversal);
@@ -109,7 +112,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest {
}
@Test
- @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ)
public void g_io_read_withXreader_graphmlX() throws IOException {
final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/');
final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphmlX(fileToRead);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
index e739c0a..55e6f9c 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java
@@ -18,12 +18,14 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.map;
+import org.apache.tinkerpop.gremlin.FeatureRequirement;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.structure.Graph;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -54,7 +56,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void g_writeXkryoX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
+ public void g_io_writeXkryoX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class, "tinkerpop-modern-v3d0", ".kryo").getAbsolutePath().replace('\\', '/');
final File f = new File(fileToWrite);
@@ -69,6 +72,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
public void g_io_write_withXwrite_gryoX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class, "tinkerpop-modern-v3d0", ".kryo").getAbsolutePath().replace('\\', '/');
@@ -84,7 +88,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void g_writeXjsonX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
+ public void g_io_writeXjsonX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern-v3d0", ".json").getAbsolutePath().replace('\\', '/');
final File f = new File(fileToWrite);
@@ -99,6 +104,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
public void g_io_write_withXwriter_graphsonX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern-v3d0", ".json").getAbsolutePath().replace('\\', '/');
@@ -114,7 +120,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void g_writeXxmlX() throws IOException {
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
+ public void g_io_writeXxmlX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern", ".xml").getAbsolutePath().replace('\\', '/');
final File f = new File(fileToWrite);
@@ -129,6 +136,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE)
public void g_io_write_withXwriter_graphmlX() throws IOException {
final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern", ".xml").getAbsolutePath().replace('\\', '/');
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
index 62937da..97fdea4 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java
@@ -18,18 +18,30 @@
*/
package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map;
+import org.apache.hadoop.mapred.InputFormat;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
+import org.apache.tinkerpop.gremlin.process.traversal.IO;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+import java.lang.reflect.Method;
+
/**
* An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the
* {@link CloneVertexProgram} for its implementation.
@@ -103,12 +115,56 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting {
}
private void configureForRead(final Graph graph) {
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat");
+ final String inputFormatClassNameOrKeyword = parameters.get(IO.reader, this::detectReader).get(0);
+ String inputFormatClassName;
+ if (inputFormatClassNameOrKeyword.equals(IO.graphson))
+ inputFormatClassName = GraphSONInputFormat.class.getName();
+ else if (inputFormatClassNameOrKeyword.equals(IO.gryo))
+ inputFormatClassName = GryoInputFormat.class.getName();
+ else if (inputFormatClassNameOrKeyword.equals(IO.graphml))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ inputFormatClassName = inputFormatClassNameOrKeyword;
+
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file);
}
private void configureForWrite(final Graph graph) {
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat");
+ final String outputFormatClassNameOrKeyword = parameters.get(IO.writer, this::detectWriter).get(0);
+ String outputFormatClassName;
+ if (outputFormatClassNameOrKeyword.equals(IO.graphson))
+ outputFormatClassName = GraphSONOutputFormat.class.getName();
+ else if (outputFormatClassNameOrKeyword.equals(IO.gryo))
+ outputFormatClassName = GryoOutputFormat.class.getName();
+ else if (outputFormatClassNameOrKeyword.equals(IO.graphml))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ outputFormatClassName = outputFormatClassNameOrKeyword;
+
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName);
graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file);
}
+
+ private String detectReader() {
+ if (file.endsWith(".kryo"))
+ return GryoInputFormat.class.getName();
+ else if (file.endsWith(".json"))
+ return GraphSONInputFormat.class.getName();
+ else if (file.endsWith(".xml"))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
+ }
+
+ private String detectWriter() {
+ if (file.endsWith(".kryo"))
+ return GryoOutputFormat.class.getName();
+ else if (file.endsWith(".json"))
+ return GraphSONOutputFormat.class.getName();
+ else if (file.endsWith(".xml"))
+ throw new IllegalStateException("GraphML is not a supported file format for OLAP");
+ else
+ throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension");
+ }
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
index 6d3899e..cbc9b07 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java
@@ -55,7 +55,7 @@ public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalS
final HadoopIoStep hadoopIoStep = new HadoopIoStep(traversal, readWriting.getFile());
hadoopIoStep.setMode(readWriting.getMode());
readWriting.getParameters().getRaw().entrySet().forEach(kv ->
- hadoopIoStep.configure(kv.getKey(), kv.getValue())
+ hadoopIoStep.configure(kv.getKey(), kv.getValue().get(0))
);
TraversalHelper.replaceStep((Step) readWriting, hadoopIoStep, traversal);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 14c5360..9ec0cfd 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -142,6 +142,22 @@ import java.util.stream.Stream;
method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
computers = {"ALL"})
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals",
+ method = "g_io_readXxmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals",
+ method = "g_io_read_withXreader_graphmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest$Traversals",
+ method = "g_io_writeXxmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
+@Graph.OptOut(
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest$Traversals",
+ method = "g_io_write_withXwriter_graphmlX",
+ reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
public final class HadoopGraph implements Graph {
public static final Logger LOGGER = LoggerFactory.getLogger(HadoopGraph.class);