You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2018/04/16 11:56:35 UTC
[1/3] tinkerpop git commit: TINKERPOP-1410 Removed test artifact of
hadoop-gremlin
Repository: tinkerpop
Updated Branches:
refs/heads/master 108b236ba -> 4e5c30e20
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java
new file mode 100644
index 0000000..a5f657d
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONRecordWriter;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoRecordWriter;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.ToyIoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.ToyPoint;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.ToyTriangle;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoVersion;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public abstract class AbstractIoRegistryCheck extends AbstractGremlinTest {
+
+ private static final int NUMBER_OF_VERTICES = 1000;
+
+ public void checkGryoV1d0IoRegistryCompliance(final HadoopGraph graph, final Class<? extends GraphComputer> graphComputerClass) throws Exception {
+ final File input = TestHelper.generateTempFile(this.getClass(), "gryo-io-registry", ".kryo");
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
+ graph.configuration().setProperty(GryoPool.CONFIG_IO_GRYO_VERSION, GryoVersion.V1_0.name());
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, input.getAbsolutePath());
+ graph.configuration().setProperty(IoRegistry.IO_REGISTRY, ToyIoRegistry.class.getCanonicalName());
+ final GryoRecordWriter writer = new GryoRecordWriter(new DataOutputStream(new FileOutputStream(input)), ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ validateIoRegistryGraph(graph, graphComputerClass, writer);
+ assertTrue(input.delete());
+ }
+
+ public void checkGryoV3d0IoRegistryCompliance(final HadoopGraph graph, final Class<? extends GraphComputer> graphComputerClass) throws Exception {
+ final File input = TestHelper.generateTempFile(this.getClass(), "gryo-io-registry", ".kryo");
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, input.getAbsolutePath());
+ graph.configuration().setProperty(GryoPool.CONFIG_IO_GRYO_VERSION, GryoVersion.V3_0.name());
+ graph.configuration().setProperty(IoRegistry.IO_REGISTRY, ToyIoRegistry.class.getCanonicalName());
+ final GryoRecordWriter writer = new GryoRecordWriter(new DataOutputStream(new FileOutputStream(input)), ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ validateIoRegistryGraph(graph, graphComputerClass, writer);
+ assertTrue(input.delete());
+ }
+
+ public void checkGraphSONIoRegistryCompliance(final HadoopGraph graph, final Class<? extends GraphComputer> graphComputerClass) throws Exception {
+ final File input = TestHelper.generateTempFile(this.getClass(), "graphson-io-registry", ".json");
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GraphSONInputFormat.class.getCanonicalName());
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GraphSONOutputFormat.class.getCanonicalName());
+ graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, input.getAbsolutePath());
+ graph.configuration().setProperty(IoRegistry.IO_REGISTRY, ToyIoRegistry.class.getCanonicalName());
+ final GraphSONRecordWriter writer = new GraphSONRecordWriter(new DataOutputStream(new FileOutputStream(input)), ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ validateIoRegistryGraph(graph, graphComputerClass, writer);
+ assertTrue(input.delete());
+ }
+
+ private void validateIoRegistryGraph(final HadoopGraph graph,
+ final Class<? extends GraphComputer> graphComputerClass,
+ final RecordWriter<NullWritable, VertexWritable> writer) throws Exception {
+
+
+ for (int i = 0; i < NUMBER_OF_VERTICES; i++) {
+ final StarGraph starGraph = StarGraph.open();
+ Vertex vertex = starGraph.addVertex(T.label, "place", T.id, i, "point", new ToyPoint(i, i * 10), "message", "I'm " + i, "triangle", new ToyTriangle(i, i * 10, i * 100));
+ vertex.addEdge("connection", starGraph.addVertex(T.id, i > 0 ? i - 1 : NUMBER_OF_VERTICES - 1));
+ writer.write(NullWritable.get(), new VertexWritable(starGraph.getStarVertex()));
+ }
+ writer.close(new TaskAttemptContextImpl(ConfUtil.makeHadoopConfiguration(graph.configuration()), new TaskAttemptID()));
+
+ // OLAP TESTING //
+ validatePointTriangles(graph.traversal().withComputer(graphComputerClass).V().project("point", "triangle").by("point").by("triangle").toList());
+ validatePointTriangles(graph.traversal().withComputer(graphComputerClass).V().out().project("point", "triangle").by("point").by("triangle").toList());
+ validatePointTriangles(graph.traversal().withComputer(graphComputerClass).V().out().out().project("point", "triangle").by("point").by("triangle").toList());
+ // OLTP TESTING //
+ validatePointTriangles(graph.traversal().V().project("point", "triangle").by("point").by("triangle").toList());
+ // HDFS TESTING //
+ /*validatePointTriangles(IteratorUtils.<Map<String, Object>>asList(IteratorUtils.<Vertex, Map<String, Object>>map(FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration())).head(graph.configuration().getInputLocation(), graph.configuration().getGraphReader()),
+ vertex -> {
+ return new HashMap<String, Object>() {{
+ put("point", vertex.value("point"));
+ put("triangle", vertex.value("triangle"));
+ }};
+ })));*/
+ }
+
+ private void validatePointTriangles(final List<Map<String, Object>> values) {
+ assertEquals(NUMBER_OF_VERTICES, values.size());
+ for (int i = 0; i < NUMBER_OF_VERTICES; i++) {
+ assertTrue(values.stream().map(m -> m.<ToyPoint>get("point")).collect(Collectors.toList()).contains(new ToyPoint(i, i * 10)));
+ assertTrue(values.stream().map(m -> m.<ToyTriangle>get("triangle")).collect(Collectors.toList()).contains(new ToyTriangle(i, i * 10, i * 100)));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java
new file mode 100644
index 0000000..7b679e2
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.ClusterCountMapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.Storage;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public abstract class AbstractStorageCheck extends AbstractGremlinTest {
+
+ public void checkHeadMethods(final Storage storage, final String inputLocation, final String outputLocation, final Class outputGraphParserClass, final Class outputMemoryParserClass) throws Exception {
+ // TEST INPUT GRAPH
+ assertFalse(storage.exists(outputLocation));
+ if (inputLocation.endsWith(".json") && storage.exists(inputLocation)) { // gryo is not text readable
+ assertEquals(6, IteratorUtils.count(storage.head(inputLocation)));
+ for (int i = 0; i < 7; i++) {
+ assertEquals(i, IteratorUtils.count(storage.head(inputLocation, i)));
+ }
+ assertEquals(6, IteratorUtils.count(storage.head(inputLocation, 10)));
+ }
+
+ ////////////////////
+
+ final ComputerResult result = graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
+ // TEST OUTPUT GRAPH
+ assertTrue(storage.exists(outputLocation));
+ assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
+ assertEquals(6, result.graph().traversal().V().count().next().longValue());
+ assertEquals(0, result.graph().traversal().E().count().next().longValue());
+ assertEquals(6, result.graph().traversal().V().values("name").count().next().longValue());
+ assertEquals(6, result.graph().traversal().V().values(PeerPressureVertexProgram.CLUSTER).count().next().longValue());
+ assertEquals(2, result.graph().traversal().V().values(PeerPressureVertexProgram.CLUSTER).dedup().count().next().longValue());
+ assertEquals(6, IteratorUtils.count(storage.head(Constants.getGraphLocation(outputLocation), outputGraphParserClass)));
+ for (int i = 0; i < 7; i++) {
+ assertEquals(i, IteratorUtils.count(storage.head(Constants.getGraphLocation(outputLocation), outputGraphParserClass, i)));
+ }
+ assertEquals(6, IteratorUtils.count(storage.head(Constants.getGraphLocation(outputLocation), outputGraphParserClass, 346)));
+ /////
+ // TEST MEMORY PERSISTENCE
+ assertEquals(2, (int) result.memory().get("clusterCount"));
+ assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
+ assertEquals(1, IteratorUtils.count(storage.head(outputLocation, "clusterCount", outputMemoryParserClass)));
+ assertEquals(2, storage.head(outputLocation, "clusterCount", outputMemoryParserClass).next().getValue());
+ }
+
+ public void checkRemoveAndListMethods(final Storage storage, final String outputLocation) throws Exception {
+ graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
+ assertTrue(storage.exists(outputLocation));
+ assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
+ assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
+ assertEquals(2, storage.ls(outputLocation).size());
+ assertTrue(storage.rm(Constants.getGraphLocation(outputLocation)));
+ assertFalse(storage.rm(Constants.getGraphLocation(outputLocation)));
+ assertEquals(1, storage.ls(outputLocation).size());
+ assertTrue(storage.rm(Constants.getMemoryLocation(outputLocation, "clusterCount")));
+ assertEquals(0, storage.ls(outputLocation).size());
+ assertFalse(storage.exists(Constants.getGraphLocation(outputLocation)));
+ assertFalse(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
+ if (storage.exists(outputLocation))
+ assertTrue(storage.rm(outputLocation));
+ assertFalse(storage.exists(outputLocation));
+
+ ////////////////
+
+ graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
+ assertTrue(storage.exists(outputLocation));
+ assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
+ assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
+ assertEquals(2, storage.ls(outputLocation).size());
+ assertTrue(storage.rm(outputLocation));
+ assertFalse(storage.exists(outputLocation));
+ assertEquals(0, storage.ls(outputLocation).size());
+ }
+
+ public void checkCopyMethods(final Storage storage, final String outputLocation, final String newOutputLocation, final Class outputGraphParserClass, final Class outputMemoryParserClass) throws Exception {
+ graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
+ assertTrue(storage.exists(outputLocation));
+ assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
+ assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
+ assertFalse(storage.exists(newOutputLocation));
+
+ assertTrue(storage.cp(outputLocation, newOutputLocation));
+ assertTrue(storage.exists(outputLocation));
+ assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
+ assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
+ assertTrue(storage.exists(newOutputLocation));
+ assertTrue(storage.exists(Constants.getGraphLocation(newOutputLocation)));
+ assertTrue(storage.exists(Constants.getMemoryLocation(newOutputLocation, "clusterCount")));
+
+ assertEquals(2, storage.ls(newOutputLocation).size());
+ assertEquals(6, IteratorUtils.count(storage.head(outputLocation, outputGraphParserClass)));
+ assertEquals(6, IteratorUtils.count(storage.head(newOutputLocation, outputGraphParserClass)));
+ assertEquals(1, IteratorUtils.count(storage.head(outputLocation, "clusterCount", outputMemoryParserClass)));
+ assertEquals(1, IteratorUtils.count(storage.head(newOutputLocation, "clusterCount", outputMemoryParserClass)));
+ }
+
+ public void checkResidualDataInStorage(final Storage storage, final String outputLocation) throws Exception {
+ final GraphTraversal<Vertex, Long> traversal = g.V().both("knows").groupCount("m").by("age").count();
+ assertEquals(4l, traversal.next().longValue());
+ assertFalse(storage.exists(outputLocation));
+ assertFalse(storage.exists(Constants.getGraphLocation(outputLocation)));
+ ///
+ assertEquals(3, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").size());
+ assertEquals(1, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").get(27).longValue());
+ assertEquals(2, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").get(29).longValue());
+ assertEquals(1, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").get(32).longValue());
+ }
+
+ public void checkFileDirectoryDistinction(final Storage storage, final String directory1, final String directory2) throws Exception {
+ assertTrue(storage.exists(directory1));
+ assertTrue(storage.exists(directory2));
+ assertTrue(storage.exists(directory1 + "/f*"));
+ assertTrue(storage.exists(directory2 + "/f*"));
+ assertEquals(10, storage.ls(directory1).size());
+ assertEquals(10, storage.ls(directory1 + "/*").size());
+ assertEquals(5, storage.ls(directory2).size());
+ assertEquals(5, storage.ls(directory2 + "/*").size());
+ for (int i = 0; i < 10; i++) {
+ assertTrue(storage.exists(directory1 + "/file1-" + i + ".txt.bz"));
+ assertTrue(storage.exists(directory1 + "/file1-" + i + "*"));
+ assertTrue(storage.exists(directory1 + "/file1-" + i + ".txt*"));
+ assertTrue(storage.exists(directory1 + "/file1-" + i + ".*.bz"));
+ assertTrue(storage.exists(directory1 + "/file1-" + i + ".*.b*"));
+ }
+ assertFalse(storage.exists(directory1 + "/file1-10.txt.bz"));
+ for (int i = 0; i < 5; i++) {
+ assertTrue(storage.exists(directory2 + "/file2-" + i + ".txt.bz"));
+ assertTrue(storage.exists(directory2 + "/file2-" + i + "*"));
+ assertTrue(storage.exists(directory2 + "/file2-" + i + ".txt*"));
+ assertTrue(storage.exists(directory2 + "/file2-" + i + ".*.bz"));
+ assertTrue(storage.exists(directory2 + "/file2-" + i + ".*.b*"));
+ }
+ assertFalse(storage.exists(directory2 + "/file1-5.txt.bz"));
+ assertTrue(storage.rm(directory1 + "/file1-0.txt.bz"));
+ assertFalse(storage.rm(directory1 + "/file1-0.txt.bz"));
+ assertEquals(9, storage.ls(directory1).size());
+ assertEquals(9, storage.ls(directory1 + "/*").size());
+ assertEquals(9, storage.ls(directory1 + "/file*").size());
+ assertEquals(9, storage.ls(directory1 + "/file1*").size());
+ assertEquals(0, storage.ls(directory1 + "/file2*").size());
+ assertEquals(5, storage.ls(directory2).size());
+ assertEquals(5, storage.ls(directory2 + "/*").size());
+ assertEquals(5, storage.ls(directory2 + "/file*").size());
+ assertEquals(5, storage.ls(directory2 + "/file2*").size());
+ assertEquals(0, storage.ls(directory2 + "/file1*").size());
+ assertTrue(storage.rm(directory1 + "/file1-*"));
+ assertFalse(storage.rm(directory1 + "/file1-*"));
+ assertEquals(0, storage.ls(directory1).size());
+ assertEquals(0, storage.ls(directory1 + "/*").size());
+ assertEquals(5, storage.ls(directory2).size());
+ assertEquals(5, storage.ls(directory2 + "/*").size());
+ assertTrue(storage.rm(directory2 + "/f*"));
+ assertFalse(storage.rm(directory2 + "/file*"));
+ assertEquals(0, storage.ls(directory2).size());
+ assertEquals(0, storage.ls(directory2 + "*").size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java
new file mode 100644
index 0000000..c2eeac3
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop.structure.io;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.structure.io.Storage;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class FileSystemStorageCheck extends AbstractStorageCheck {
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportHeadMethods() throws Exception {
+ final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ final String inputLocation = Constants.getSearchGraphLocation(graph.configuration().getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION), storage).get();
+ final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
+ // TestHelper creates the directory and we need it not to exist
+ deleteDirectory(outputLocation);
+ super.checkHeadMethods(storage, inputLocation, outputLocation, InputOutputHelper.getInputFormat((Class) Class.forName(graph.configuration().getString(Constants.GREMLIN_HADOOP_GRAPH_WRITER))), SequenceFileInputFormat.class);
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoveAndListMethods() throws Exception {
+ final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
+ super.checkRemoveAndListMethods(storage, outputLocation);
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportCopyMethods() throws Exception {
+ final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
+ final String newOutputLocation = TestHelper.makeTestDataDirectory(FileSystemStorageCheck.class, "new-location-for-copy");
+ // TestHelper creates the directory and we need it not to exist
+ deleteDirectory(newOutputLocation);
+ super.checkCopyMethods(storage, outputLocation, newOutputLocation, InputOutputHelper.getInputFormat((Class) Class.forName(graph.configuration().getString(Constants.GREMLIN_HADOOP_GRAPH_WRITER))), SequenceFileInputFormat.class);
+
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldNotHaveResidualDataInStorage() throws Exception {
+ final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
+ super.checkResidualDataInStorage(storage, outputLocation);
+ }
+
+ @Test
+ public void shouldSupportDirectoryFileDistinction() throws Exception {
+ final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
+ final String directory1 = TestHelper.makeTestDataDirectory(FileSystemStorageCheck.class, "directory1");
+ final String directory2 = TestHelper.makeTestDataDirectory(FileSystemStorageCheck.class, "directory2");
+ for (int i = 0; i < 10; i++) {
+ new File(directory1 + "/" + "file1-" + i + ".txt.bz").createNewFile();
+ }
+ for (int i = 0; i < 5; i++) {
+ new File(directory2 + "/" + "file2-" + i + ".txt.bz").createNewFile();
+ }
+ super.checkFileDirectoryDistinction(storage, directory1, directory2);
+ deleteDirectory(directory1);
+ deleteDirectory(directory2);
+ }
+
+ private static void deleteDirectory(final String location) throws IOException {
+ // TestHelper creates the directory and we need it not to exist
+ assertTrue(new File(location).isDirectory());
+ assertTrue(new File(location).exists());
+ FileUtils.deleteDirectory(new File(location));
+ assertFalse(new File(location).exists());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 051d5f0..c778c6d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -20,14 +20,24 @@ package org.apache.tinkerpop.gremlin.spark.process.computer;
import org.apache.spark.launcher.SparkLauncher;
import org.apache.spark.serializer.KryoSerializer;
+import org.apache.tinkerpop.gremlin.AbstractFileGraphProvider;
import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorageCheck;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.computer.Computer;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PeerPressureTest;
@@ -40,18 +50,50 @@ import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Stephen Mallette (http://stephen.genoprime.com)
*/
@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
-public class SparkHadoopGraphProvider extends HadoopGraphProvider {
+public class SparkHadoopGraphProvider extends AbstractFileGraphProvider {
- protected static final String PREVIOUS_SPARK_PROVIDER = "previous.spark.provider";
+ static final String PREVIOUS_SPARK_PROVIDER = "previous.spark.provider";
+
+ public static final Set<Class> IMPLEMENTATION = Collections.unmodifiableSet(new HashSet<Class>() {{
+ add(HadoopEdge.class);
+ add(HadoopElement.class);
+ add(HadoopGraph.class);
+ add(HadoopProperty.class);
+ add(HadoopVertex.class);
+ add(HadoopVertexProperty.class);
+ add(ComputerGraph.class);
+ add(ComputerGraph.ComputerElement.class);
+ add(ComputerGraph.ComputerVertex.class);
+ add(ComputerGraph.ComputerEdge.class);
+ add(ComputerGraph.ComputerVertexProperty.class);
+ add(ComputerGraph.ComputerAdjacentVertex.class);
+ add(ComputerGraph.ComputerProperty.class);
+ }});
+
+ @Override
+ public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
+ if (loadGraphWith != null) ((HadoopGraph) graph).configuration().setInputLocation(getInputLocation(graph, loadGraphWith.value()));
+ }
+
+ @Override
+ public Set<Class> getImplementations() {
+ return IMPLEMENTATION;
+ }
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ this.graphSONInput = RANDOM.nextBoolean();
if (this.getClass().equals(SparkHadoopGraphProvider.class) && !SparkHadoopGraphProvider.class.getCanonicalName().equals(System.getProperty(PREVIOUS_SPARK_PROVIDER, null))) {
Spark.close();
HadoopPools.close();
@@ -59,8 +101,15 @@ public class SparkHadoopGraphProvider extends HadoopGraphProvider {
System.setProperty(PREVIOUS_SPARK_PROVIDER, SparkHadoopGraphProvider.class.getCanonicalName());
}
- final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
+ final Map<String,Object> config = new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_READER, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, getWorkingDirectory());
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+
+ put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast
+ }};
// toy graph inputRDD does not have corresponding outputRDD so where jobs chain, it fails (failing makes sense)
if (null != loadGraphWith &&
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/util/SugarTestHelper.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/util/SugarTestHelper.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/util/SugarTestHelper.java
index 334c67f..304f05f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/util/SugarTestHelper.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/util/SugarTestHelper.java
@@ -19,11 +19,11 @@
package org.apache.tinkerpop.gremlin.spark.util;
import org.apache.tinkerpop.gremlin.groovy.util.MetaRegistryUtil;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.DefaultGraphTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.*;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkHadoopGraphProvider;
import java.util.HashSet;
import java.util.Set;
@@ -49,7 +49,7 @@ public final class SugarTestHelper {
*/
public static void clearRegistry() {
final Set<Class> implementationsToClear = new HashSet<>(CORE_IMPLEMENTATIONS);
- implementationsToClear.addAll(HadoopGraphProvider.IMPLEMENTATION);
+ implementationsToClear.addAll(SparkHadoopGraphProvider.IMPLEMENTATION);
MetaRegistryUtil.clearRegistry(implementationsToClear);
}
[2/3] tinkerpop git commit: TINKERPOP-1410 Removed test artifact of
hadoop-gremlin
Posted by sp...@apache.org.
TINKERPOP-1410 Removed test artifact of hadoop-gremlin
The hadoop-gremlin test artifact allowed us to re-use some classes in spark and giraph testing, but (1) we probably shouldn't have done that in the first place as it's not a great practice and (2) we no longer have giraph and there really aren't any providers that rely on these classes that I'm aware of. I moved around a few classes to get this to all work which is described in more detail in upgrade docs. All the test still execute as they did before.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7eec250a
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7eec250a
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7eec250a
Branch: refs/heads/master
Commit: 7eec250a45a4a65bec87364d2c6ca8d895c7cb10
Parents: c14aa30
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Apr 6 13:10:22 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Apr 6 13:10:22 2018 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/upgrade/release-3.4.x.asciidoc | 22 +++
.../gremlin/AbstractFileGraphProvider.java | 110 +++++++++++
.../structure/io/gryo/ToyIoRegistry.java | 69 +++++++
.../gremlin/structure/io/gryo/ToyPoint.java | 109 +++++++++++
.../gremlin/structure/io/gryo/ToyTriangle.java | 119 ++++++++++++
hadoop-gremlin/pom.xml | 8 -
.../gremlin/hadoop/HadoopGraphProvider.java | 85 +--------
.../gremlin/hadoop/HadoopGremlinSuite.java | 35 ----
.../structure/io/AbstractIoRegistryCheck.java | 137 -------------
.../structure/io/AbstractStorageCheck.java | 190 -------------------
.../structure/io/FileSystemStorageCheck.java | 104 ----------
.../hadoop/structure/io/gryo/ToyIoRegistry.java | 70 -------
.../hadoop/structure/io/gryo/ToyPoint.java | 113 -----------
.../hadoop/structure/io/gryo/ToyTriangle.java | 120 ------------
spark-gremlin/pom.xml | 57 ------
.../gremlin/hadoop/HadoopGremlinSuite.java | 35 ++++
.../structure/io/AbstractIoRegistryCheck.java | 136 +++++++++++++
.../structure/io/AbstractStorageCheck.java | 190 +++++++++++++++++++
.../structure/io/FileSystemStorageCheck.java | 104 ++++++++++
.../computer/SparkHadoopGraphProvider.java | 59 +++++-
.../gremlin/spark/util/SugarTestHelper.java | 4 +-
22 files changed, 955 insertions(+), 922 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index fcc96b2..c577eea 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ NEED AND IMAGE
This release also includes changes from <<release-3-3-2, 3.3.2>>.
* Change the `toString()` of `Path` to be standardized as other graph elements are.
+* `hadoop-gremlin` no longer generates a test artifact.
* Fixed a bug in `ReducingBarrierStep`, that returned the provided seed value despite no elements being available.
* Changed the order of `select()` scopes. The order is now: maps, side-effects, paths.
* Removed support for Giraph.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/docs/src/upgrade/release-3.4.x.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.4.x.asciidoc b/docs/src/upgrade/release-3.4.x.asciidoc
index 713916a..bdf443e 100644
--- a/docs/src/upgrade/release-3.4.x.asciidoc
+++ b/docs/src/upgrade/release-3.4.x.asciidoc
@@ -122,3 +122,25 @@ gremlin> g.V(1).
----
See: link:https://issues.apache.org/jira/browse/TINKERPOP-1522[TINKERPOP-1522]
+
+=== Upgrading for Providers
+
+==== Graph Providers
+
+===== Removed hadoop-gremlin Test Artifact
+
+The `hadoop-gremlin` module no longer generates a test jar that can be used as a test dependency in other modules.
+Generally speaking, that approach tends to be a bad practice and can cause build problems with Maven that aren't always
+obvious to troubleshoot. With the removal of `giraph-gremlin` for 3.4.0, it seemed even less useful to have this
+test artifact present. All tests are still present. The follow provides a basic summary of how this refactoring
+occurred:
+
+* A new `AbstractFileGraphProvider` was created in `gremlin-test` which provided a lot of the features that
+`HadoopGraphProvider` was exposing. Both `HadoopGraphProvider` and `SparkHadoopGraphProvider` extend from that class
+now.
+* `ToyIoRegistry` and related classes were moved to `gremlin-test`.
+* The various tests that validated capabilities of `Storage` have been moved to `spark-gremlin` and are part of those
+tests now. Obviously, that makes those tests specific to Spark testing now. If that location creates a problem for some
+reason, that decision can be revisited at some point.
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-1410[TINKERPOP-1410]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
new file mode 100644
index 0000000..d16b289
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+/**
+ * A base {@link GraphProvider} that is typically for use with Hadoop-based graphs as it enables access to the various
+ * resource data files that are used in the tests.
+ *
+ * @author Stephen Mallette (http://stephen.genoprime.com)
+ */
+public abstract class AbstractFileGraphProvider extends AbstractGraphProvider {
+
+ protected static final Random RANDOM = new Random();
+
+ protected boolean graphSONInput = false;
+
+ public static Map<String, String> PATHS = new HashMap<>();
+ static {
+ try {
+ final List<String> kryoResources = Arrays.asList(
+ "tinkerpop-modern-v3d0.kryo",
+ "grateful-dead-v3d0.kryo",
+ "tinkerpop-classic-v3d0.kryo",
+ "tinkerpop-crew-v3d0.kryo",
+ "tinkerpop-sink-v3d0.kryo");
+ for (final String fileName : kryoResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
+ }
+
+ final List<String> graphsonResources = Arrays.asList(
+ "tinkerpop-modern-typed-v2d0.json",
+ "tinkerpop-modern-v3d0.json",
+ "grateful-dead-typed-v2d0.json",
+ "grateful-dead-v3d0.json",
+ "tinkerpop-classic-typed-v2d0.json",
+ "tinkerpop-classic-v3d0.json",
+ "tinkerpop-crew-typed-v2d0.json",
+ "tinkerpop-crew-v3d0.json",
+ "tinkerpop-sink-v3d0.json");
+ for (final String fileName : graphsonResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
+ }
+
+ final List<String> scriptResources = Arrays.asList(
+ "tinkerpop-classic.txt",
+ "script-input.groovy",
+ "script-output.groovy",
+ "grateful-dead.txt",
+ "script-input-grateful-dead.groovy",
+ "script-output-grateful-dead.groovy");
+ for (final String fileName : scriptResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void clear(final Graph graph, final Configuration configuration) throws Exception {
+ if (graph != null)
+ graph.close();
+ }
+
+ protected String getInputLocation(final Graph g, final LoadGraphWith.GraphData graphData) {
+ final String type = this.graphSONInput ? "-v3d0.json" : "-v3d0.kryo";
+
+ if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL))
+ return PATHS.get("grateful-dead" + type);
+ else if (graphData.equals(LoadGraphWith.GraphData.MODERN))
+ return PATHS.get("tinkerpop-modern" + type);
+ else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC))
+ return PATHS.get("tinkerpop-classic" + type);
+ else if (graphData.equals(LoadGraphWith.GraphData.CREW))
+ return PATHS.get("tinkerpop-crew" + type);
+ else if (graphData.equals(LoadGraphWith.GraphData.SINK))
+ return PATHS.get("tinkerpop-sink" + type);
+ else
+ throw new RuntimeException("Could not load graph with " + graphData);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyIoRegistry.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyIoRegistry.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyIoRegistry.java
new file mode 100644
index 0000000..339a83d
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyIoRegistry.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.structure.io.gryo;
+
+import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ToyIoRegistry extends AbstractIoRegistry {
+
+ private static final ToyIoRegistry INSTANCE = new ToyIoRegistry();
+
+ private ToyIoRegistry() {
+ super.register(GryoIo.class, ToyPoint.class, new ToyPoint.ToyPointSerializer());
+ super.register(GryoIo.class, ToyTriangle.class, new ToyTriangle.ToyTriangleSerializer());
+ super.register(GraphSONIo.class, null, new ToyModule());
+ }
+
+ public static class ToyModule extends TinkerPopJacksonModule {
+ public ToyModule() {
+ super("toy");
+ addSerializer(ToyPoint.class, new ToyPoint.ToyPointJacksonSerializer());
+ addDeserializer(ToyPoint.class, new ToyPoint.ToyPointJacksonDeSerializer());
+ addSerializer(ToyTriangle.class, new ToyTriangle.ToyTriangleJacksonSerializer());
+ addDeserializer(ToyTriangle.class, new ToyTriangle.ToyTriangleJacksonDeSerializer());
+ }
+
+
+ @Override
+ public Map<Class, String> getTypeDefinitions() {
+ return new HashMap<Class, String>() {{
+ put(ToyPoint.class, "ToyPoint");
+ put(ToyTriangle.class, "ToyTriangle");
+ }};
+ }
+
+ @Override
+ public String getTypeNamespace() {
+ return "toy";
+ }
+ }
+
+ public static ToyIoRegistry instance() {
+ return INSTANCE;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyPoint.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyPoint.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyPoint.java
new file mode 100644
index 0000000..3bab073
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyPoint.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.structure.io.gryo;
+
+import org.apache.tinkerpop.gremlin.structure.io.graphson.AbstractObjectDeserializer;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException;
+import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
+import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider;
+import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ToyPoint {
+
+ private final int x;
+ private final int y;
+
+ public ToyPoint(final int x, final int y) {
+ this.x = x;
+ this.y = y;
+ }
+
+ public int getX() {
+ return this.x;
+ }
+
+ public int getY() {
+ return this.y;
+ }
+
+ public int hashCode() {
+ return this.x + this.y;
+ }
+
+ public boolean equals(final Object other) {
+ return other instanceof ToyPoint && ((ToyPoint) other).x == this.x && ((ToyPoint) other).y == this.y;
+ }
+
+ @Override
+ public String toString() {
+ return "[" + this.x + "," + this.y + "]";
+ }
+
+ public static class ToyPointSerializer implements SerializerShim<ToyPoint> {
+ @Override
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ToyPoint toyPoint) {
+ output.writeInt(toyPoint.x);
+ output.writeInt(toyPoint.y);
+ }
+
+ @Override
+ public <I extends InputShim> ToyPoint read(final KryoShim<I, ?> kryo, final I input, final Class<ToyPoint> toyPointClass) {
+ return new ToyPoint(input.readInt(), input.readInt());
+ }
+ }
+
+ public static class ToyPointJacksonSerializer extends StdScalarSerializer<ToyPoint> {
+
+ public ToyPointJacksonSerializer() {
+ super(ToyPoint.class);
+ }
+
+ @Override
+ public void serialize(final ToyPoint toyPoint, final JsonGenerator jsonGenerator,
+ final SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
+ jsonGenerator.writeStartObject();
+ jsonGenerator.writeObjectField("x", toyPoint.x);
+ jsonGenerator.writeObjectField("y", toyPoint.y);
+ jsonGenerator.writeEndObject();
+ }
+ }
+
+ public static class ToyPointJacksonDeSerializer extends AbstractObjectDeserializer<ToyPoint> {
+
+ public ToyPointJacksonDeSerializer() {
+ super(ToyPoint.class);
+ }
+
+ @Override
+ public ToyPoint createObject(final Map<String, Object> map) {
+ return new ToyPoint((int) map.get("x"), (int) map.get("y"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyTriangle.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyTriangle.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyTriangle.java
new file mode 100644
index 0000000..614ae70
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/ToyTriangle.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.structure.io.gryo;
+
+import org.apache.tinkerpop.gremlin.structure.io.graphson.AbstractObjectDeserializer;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
+import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException;
+import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
+import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider;
+import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ToyTriangle {
+
+ private final int x;
+ private final int y;
+ private final int z;
+
+ public ToyTriangle(final int x, final int y, final int z) {
+ this.x = x;
+ this.y = y;
+ this.z = z;
+ }
+
+ public int getX() {
+ return this.x;
+ }
+
+ public int getY() {
+ return this.y;
+ }
+
+ public int getZ() {
+ return this.z;
+ }
+
+ public int hashCode() {
+ return this.x + this.y + this.z;
+ }
+
+ public boolean equals(final Object other) {
+ return other instanceof ToyTriangle && ((ToyTriangle) other).x == this.x && ((ToyTriangle) other).y == this.y && ((ToyTriangle) other).z == this.z;
+ }
+
+ @Override
+ public String toString() {
+ return "[" + this.x + "," + this.y + "," + this.z + "]";
+ }
+
+ public static class ToyTriangleSerializer implements SerializerShim<ToyTriangle> {
+ @Override
+ public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ToyTriangle toyTriangle) {
+ output.writeInt(toyTriangle.x);
+ output.writeInt(toyTriangle.y);
+ output.writeInt(toyTriangle.z);
+ }
+
+ @Override
+ public <I extends InputShim> ToyTriangle read(final KryoShim<I, ?> kryo, final I input, final Class<ToyTriangle> toyTriangleClass) {
+ return new ToyTriangle(input.readInt(), input.readInt(), input.readInt());
+ }
+ }
+
+
+ public static class ToyTriangleJacksonSerializer extends StdScalarSerializer<ToyTriangle> {
+
+ public ToyTriangleJacksonSerializer() {
+ super(ToyTriangle.class);
+ }
+
+ @Override
+ public void serialize(final ToyTriangle toyTriangle, final JsonGenerator jsonGenerator,
+ final SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
+ jsonGenerator.writeStartObject();
+ jsonGenerator.writeObjectField("x", toyTriangle.x);
+ jsonGenerator.writeObjectField("y", toyTriangle.y);
+ jsonGenerator.writeObjectField("z", toyTriangle.z);
+ jsonGenerator.writeEndObject();
+ }
+ }
+
+ public static class ToyTriangleJacksonDeSerializer extends AbstractObjectDeserializer<ToyTriangle> {
+
+ public ToyTriangleJacksonDeSerializer() {
+ super(ToyTriangle.class);
+ }
+
+ @Override
+ public ToyTriangle createObject(final Map<String, Object> map) {
+ return new ToyTriangle((int) map.get("x"), (int) map.get("y"), (int) map.get("z"));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index 210a0ec..6666834 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -174,14 +174,6 @@ limitations under the License.
</manifestEntries>
</archive>
</configuration>
- <executions>
- <execution>
- <id>test-jar-it-up</id>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index 92c361f..647e004 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -18,10 +18,8 @@
*/
package org.apache.tinkerpop.gremlin.hadoop;
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
+import org.apache.tinkerpop.gremlin.AbstractFileGraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
@@ -33,17 +31,11 @@ import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
/**
@@ -51,13 +43,9 @@ import java.util.Set;
* @author Stephen Mallette (http://stephen.genoprime.com)
* @author Daniel Kuppitz (http://gremlin.guru)
*/
-public class HadoopGraphProvider extends AbstractGraphProvider {
+public class HadoopGraphProvider extends AbstractFileGraphProvider {
- protected static final Random RANDOM = new Random();
- private boolean graphSONInput = false;
-
- public static Map<String, String> PATHS = new HashMap<>();
- public static final Set<Class> IMPLEMENTATION = Collections.unmodifiableSet(new HashSet<Class>() {{
+ private static final Set<Class> IMPLEMENTATION = Collections.unmodifiableSet(new HashSet<Class>() {{
add(HadoopEdge.class);
add(HadoopElement.class);
add(HadoopGraph.class);
@@ -73,47 +61,6 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
add(ComputerGraph.ComputerProperty.class);
}});
- static {
- try {
- final List<String> kryoResources = Arrays.asList(
- "tinkerpop-modern-v3d0.kryo",
- "grateful-dead-v3d0.kryo",
- "tinkerpop-classic-v3d0.kryo",
- "tinkerpop-crew-v3d0.kryo",
- "tinkerpop-sink-v3d0.kryo");
- for (final String fileName : kryoResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
- }
-
- final List<String> graphsonResources = Arrays.asList(
- "tinkerpop-modern-typed-v2d0.json",
- "tinkerpop-modern-v3d0.json",
- "grateful-dead-typed-v2d0.json",
- "grateful-dead-v3d0.json",
- "tinkerpop-classic-typed-v2d0.json",
- "tinkerpop-classic-v3d0.json",
- "tinkerpop-crew-typed-v2d0.json",
- "tinkerpop-crew-v3d0.json",
- "tinkerpop-sink-v3d0.json");
- for (final String fileName : graphsonResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
- }
-
- final List<String> scriptResources = Arrays.asList(
- "tinkerpop-classic.txt",
- "script-input.groovy",
- "script-output.groovy",
- "grateful-dead.txt",
- "script-input-grateful-dead.groovy",
- "script-output-grateful-dead.groovy");
- for (final String fileName : scriptResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/'));
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
this.graphSONInput = RANDOM.nextBoolean();
@@ -127,36 +74,12 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
}
@Override
- public void clear(final Graph graph, final Configuration configuration) throws Exception {
- if (graph != null)
- graph.close();
- }
-
- @Override
public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
- if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
+ if (loadGraphWith != null) ((HadoopGraph) graph).configuration().setInputLocation(getInputLocation(graph, loadGraphWith.value()));
}
@Override
public Set<Class> getImplementations() {
return IMPLEMENTATION;
}
-
- public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
- final String type = this.graphSONInput ? "-v3d0.json" : "-v3d0.kryo";
-
- if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead" + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern" + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic" + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew" + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.SINK)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-sink" + type));
- } else {
- throw new RuntimeException("Could not load graph with " + graphData);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java
deleted file mode 100644
index 2bce560..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop;
-
-import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorageCheck;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
-import org.junit.runners.model.InitializationError;
-import org.junit.runners.model.RunnerBuilder;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class HadoopGremlinSuite extends AbstractGremlinSuite {
- public HadoopGremlinSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
- super(klass, builder, new Class<?>[]{FileSystemStorageCheck.class}, new Class<?>[]{FileSystemStorageCheck.class}, true, TraversalEngine.Type.COMPUTER);
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java
deleted file mode 100644
index 8b89fc1..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractIoRegistryCheck.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-
-
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONRecordWriter;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoRecordWriter;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.ToyIoRegistry;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.ToyPoint;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.ToyTriangle;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.structure.T;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.io.IoRegistry;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoVersion;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public abstract class AbstractIoRegistryCheck extends AbstractGremlinTest {
-
- private static final int NUMBER_OF_VERTICES = 1000;
-
- public void checkGryoV1d0IoRegistryCompliance(final HadoopGraph graph, final Class<? extends GraphComputer> graphComputerClass) throws Exception {
- final File input = TestHelper.generateTempFile(this.getClass(), "gryo-io-registry", ".kryo");
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
- graph.configuration().setProperty(GryoPool.CONFIG_IO_GRYO_VERSION, GryoVersion.V1_0.name());
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, input.getAbsolutePath());
- graph.configuration().setProperty(IoRegistry.IO_REGISTRY, ToyIoRegistry.class.getCanonicalName());
- final GryoRecordWriter writer = new GryoRecordWriter(new DataOutputStream(new FileOutputStream(input)), ConfUtil.makeHadoopConfiguration(graph.configuration()));
- validateIoRegistryGraph(graph, graphComputerClass, writer);
- assertTrue(input.delete());
- }
-
- public void checkGryoV3d0IoRegistryCompliance(final HadoopGraph graph, final Class<? extends GraphComputer> graphComputerClass) throws Exception {
- final File input = TestHelper.generateTempFile(this.getClass(), "gryo-io-registry", ".kryo");
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, input.getAbsolutePath());
- graph.configuration().setProperty(GryoPool.CONFIG_IO_GRYO_VERSION, GryoVersion.V3_0.name());
- graph.configuration().setProperty(IoRegistry.IO_REGISTRY, ToyIoRegistry.class.getCanonicalName());
- final GryoRecordWriter writer = new GryoRecordWriter(new DataOutputStream(new FileOutputStream(input)), ConfUtil.makeHadoopConfiguration(graph.configuration()));
- validateIoRegistryGraph(graph, graphComputerClass, writer);
- assertTrue(input.delete());
- }
-
- public void checkGraphSONIoRegistryCompliance(final HadoopGraph graph, final Class<? extends GraphComputer> graphComputerClass) throws Exception {
- final File input = TestHelper.generateTempFile(this.getClass(), "graphson-io-registry", ".json");
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GraphSONInputFormat.class.getCanonicalName());
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GraphSONOutputFormat.class.getCanonicalName());
- graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, input.getAbsolutePath());
- graph.configuration().setProperty(IoRegistry.IO_REGISTRY, ToyIoRegistry.class.getCanonicalName());
- final GraphSONRecordWriter writer = new GraphSONRecordWriter(new DataOutputStream(new FileOutputStream(input)), ConfUtil.makeHadoopConfiguration(graph.configuration()));
- validateIoRegistryGraph(graph, graphComputerClass, writer);
- assertTrue(input.delete());
- }
-
- private void validateIoRegistryGraph(final HadoopGraph graph,
- final Class<? extends GraphComputer> graphComputerClass,
- final RecordWriter<NullWritable, VertexWritable> writer) throws Exception {
-
-
- for (int i = 0; i < NUMBER_OF_VERTICES; i++) {
- final StarGraph starGraph = StarGraph.open();
- Vertex vertex = starGraph.addVertex(T.label, "place", T.id, i, "point", new ToyPoint(i, i * 10), "message", "I'm " + i, "triangle", new ToyTriangle(i, i * 10, i * 100));
- vertex.addEdge("connection", starGraph.addVertex(T.id, i > 0 ? i - 1 : NUMBER_OF_VERTICES - 1));
- writer.write(NullWritable.get(), new VertexWritable(starGraph.getStarVertex()));
- }
- writer.close(new TaskAttemptContextImpl(ConfUtil.makeHadoopConfiguration(graph.configuration()), new TaskAttemptID()));
-
- // OLAP TESTING //
- validatePointTriangles(graph.traversal().withComputer(graphComputerClass).V().project("point", "triangle").by("point").by("triangle").toList());
- validatePointTriangles(graph.traversal().withComputer(graphComputerClass).V().out().project("point", "triangle").by("point").by("triangle").toList());
- validatePointTriangles(graph.traversal().withComputer(graphComputerClass).V().out().out().project("point", "triangle").by("point").by("triangle").toList());
- // OLTP TESTING //
- validatePointTriangles(graph.traversal().V().project("point", "triangle").by("point").by("triangle").toList());
- // HDFS TESTING //
- /*validatePointTriangles(IteratorUtils.<Map<String, Object>>asList(IteratorUtils.<Vertex, Map<String, Object>>map(FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration())).head(graph.configuration().getInputLocation(), graph.configuration().getGraphReader()),
- vertex -> {
- return new HashMap<String, Object>() {{
- put("point", vertex.value("point"));
- put("triangle", vertex.value("triangle"));
- }};
- })));*/
- }
-
- private void validatePointTriangles(final List<Map<String, Object>> values) {
- assertEquals(NUMBER_OF_VERTICES, values.size());
- for (int i = 0; i < NUMBER_OF_VERTICES; i++) {
- assertTrue(values.stream().map(m -> m.<ToyPoint>get("point")).collect(Collectors.toList()).contains(new ToyPoint(i, i * 10)));
- assertTrue(values.stream().map(m -> m.<ToyTriangle>get("triangle")).collect(Collectors.toList()).contains(new ToyTriangle(i, i * 10, i * 100)));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java
deleted file mode 100644
index 7b679e2..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/AbstractStorageCheck.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-
-import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.ClusterCountMapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgram;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.io.Storage;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public abstract class AbstractStorageCheck extends AbstractGremlinTest {
-
- public void checkHeadMethods(final Storage storage, final String inputLocation, final String outputLocation, final Class outputGraphParserClass, final Class outputMemoryParserClass) throws Exception {
- // TEST INPUT GRAPH
- assertFalse(storage.exists(outputLocation));
- if (inputLocation.endsWith(".json") && storage.exists(inputLocation)) { // gryo is not text readable
- assertEquals(6, IteratorUtils.count(storage.head(inputLocation)));
- for (int i = 0; i < 7; i++) {
- assertEquals(i, IteratorUtils.count(storage.head(inputLocation, i)));
- }
- assertEquals(6, IteratorUtils.count(storage.head(inputLocation, 10)));
- }
-
- ////////////////////
-
- final ComputerResult result = graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
- // TEST OUTPUT GRAPH
- assertTrue(storage.exists(outputLocation));
- assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
- assertEquals(6, result.graph().traversal().V().count().next().longValue());
- assertEquals(0, result.graph().traversal().E().count().next().longValue());
- assertEquals(6, result.graph().traversal().V().values("name").count().next().longValue());
- assertEquals(6, result.graph().traversal().V().values(PeerPressureVertexProgram.CLUSTER).count().next().longValue());
- assertEquals(2, result.graph().traversal().V().values(PeerPressureVertexProgram.CLUSTER).dedup().count().next().longValue());
- assertEquals(6, IteratorUtils.count(storage.head(Constants.getGraphLocation(outputLocation), outputGraphParserClass)));
- for (int i = 0; i < 7; i++) {
- assertEquals(i, IteratorUtils.count(storage.head(Constants.getGraphLocation(outputLocation), outputGraphParserClass, i)));
- }
- assertEquals(6, IteratorUtils.count(storage.head(Constants.getGraphLocation(outputLocation), outputGraphParserClass, 346)));
- /////
- // TEST MEMORY PERSISTENCE
- assertEquals(2, (int) result.memory().get("clusterCount"));
- assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
- assertEquals(1, IteratorUtils.count(storage.head(outputLocation, "clusterCount", outputMemoryParserClass)));
- assertEquals(2, storage.head(outputLocation, "clusterCount", outputMemoryParserClass).next().getValue());
- }
-
- public void checkRemoveAndListMethods(final Storage storage, final String outputLocation) throws Exception {
- graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
- assertTrue(storage.exists(outputLocation));
- assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
- assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
- assertEquals(2, storage.ls(outputLocation).size());
- assertTrue(storage.rm(Constants.getGraphLocation(outputLocation)));
- assertFalse(storage.rm(Constants.getGraphLocation(outputLocation)));
- assertEquals(1, storage.ls(outputLocation).size());
- assertTrue(storage.rm(Constants.getMemoryLocation(outputLocation, "clusterCount")));
- assertEquals(0, storage.ls(outputLocation).size());
- assertFalse(storage.exists(Constants.getGraphLocation(outputLocation)));
- assertFalse(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
- if (storage.exists(outputLocation))
- assertTrue(storage.rm(outputLocation));
- assertFalse(storage.exists(outputLocation));
-
- ////////////////
-
- graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
- assertTrue(storage.exists(outputLocation));
- assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
- assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
- assertEquals(2, storage.ls(outputLocation).size());
- assertTrue(storage.rm(outputLocation));
- assertFalse(storage.exists(outputLocation));
- assertEquals(0, storage.ls(outputLocation).size());
- }
-
- public void checkCopyMethods(final Storage storage, final String outputLocation, final String newOutputLocation, final Class outputGraphParserClass, final Class outputMemoryParserClass) throws Exception {
- graphProvider.getGraphComputer(graph).program(PeerPressureVertexProgram.build().create(graph)).mapReduce(ClusterCountMapReduce.build().memoryKey("clusterCount").create()).submit().get();
- assertTrue(storage.exists(outputLocation));
- assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
- assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
- assertFalse(storage.exists(newOutputLocation));
-
- assertTrue(storage.cp(outputLocation, newOutputLocation));
- assertTrue(storage.exists(outputLocation));
- assertTrue(storage.exists(Constants.getGraphLocation(outputLocation)));
- assertTrue(storage.exists(Constants.getMemoryLocation(outputLocation, "clusterCount")));
- assertTrue(storage.exists(newOutputLocation));
- assertTrue(storage.exists(Constants.getGraphLocation(newOutputLocation)));
- assertTrue(storage.exists(Constants.getMemoryLocation(newOutputLocation, "clusterCount")));
-
- assertEquals(2, storage.ls(newOutputLocation).size());
- assertEquals(6, IteratorUtils.count(storage.head(outputLocation, outputGraphParserClass)));
- assertEquals(6, IteratorUtils.count(storage.head(newOutputLocation, outputGraphParserClass)));
- assertEquals(1, IteratorUtils.count(storage.head(outputLocation, "clusterCount", outputMemoryParserClass)));
- assertEquals(1, IteratorUtils.count(storage.head(newOutputLocation, "clusterCount", outputMemoryParserClass)));
- }
-
- public void checkResidualDataInStorage(final Storage storage, final String outputLocation) throws Exception {
- final GraphTraversal<Vertex, Long> traversal = g.V().both("knows").groupCount("m").by("age").count();
- assertEquals(4l, traversal.next().longValue());
- assertFalse(storage.exists(outputLocation));
- assertFalse(storage.exists(Constants.getGraphLocation(outputLocation)));
- ///
- assertEquals(3, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").size());
- assertEquals(1, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").get(27).longValue());
- assertEquals(2, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").get(29).longValue());
- assertEquals(1, traversal.asAdmin().getSideEffects().<Map<Integer, Long>>get("m").get(32).longValue());
- }
-
- public void checkFileDirectoryDistinction(final Storage storage, final String directory1, final String directory2) throws Exception {
- assertTrue(storage.exists(directory1));
- assertTrue(storage.exists(directory2));
- assertTrue(storage.exists(directory1 + "/f*"));
- assertTrue(storage.exists(directory2 + "/f*"));
- assertEquals(10, storage.ls(directory1).size());
- assertEquals(10, storage.ls(directory1 + "/*").size());
- assertEquals(5, storage.ls(directory2).size());
- assertEquals(5, storage.ls(directory2 + "/*").size());
- for (int i = 0; i < 10; i++) {
- assertTrue(storage.exists(directory1 + "/file1-" + i + ".txt.bz"));
- assertTrue(storage.exists(directory1 + "/file1-" + i + "*"));
- assertTrue(storage.exists(directory1 + "/file1-" + i + ".txt*"));
- assertTrue(storage.exists(directory1 + "/file1-" + i + ".*.bz"));
- assertTrue(storage.exists(directory1 + "/file1-" + i + ".*.b*"));
- }
- assertFalse(storage.exists(directory1 + "/file1-10.txt.bz"));
- for (int i = 0; i < 5; i++) {
- assertTrue(storage.exists(directory2 + "/file2-" + i + ".txt.bz"));
- assertTrue(storage.exists(directory2 + "/file2-" + i + "*"));
- assertTrue(storage.exists(directory2 + "/file2-" + i + ".txt*"));
- assertTrue(storage.exists(directory2 + "/file2-" + i + ".*.bz"));
- assertTrue(storage.exists(directory2 + "/file2-" + i + ".*.b*"));
- }
- assertFalse(storage.exists(directory2 + "/file1-5.txt.bz"));
- assertTrue(storage.rm(directory1 + "/file1-0.txt.bz"));
- assertFalse(storage.rm(directory1 + "/file1-0.txt.bz"));
- assertEquals(9, storage.ls(directory1).size());
- assertEquals(9, storage.ls(directory1 + "/*").size());
- assertEquals(9, storage.ls(directory1 + "/file*").size());
- assertEquals(9, storage.ls(directory1 + "/file1*").size());
- assertEquals(0, storage.ls(directory1 + "/file2*").size());
- assertEquals(5, storage.ls(directory2).size());
- assertEquals(5, storage.ls(directory2 + "/*").size());
- assertEquals(5, storage.ls(directory2 + "/file*").size());
- assertEquals(5, storage.ls(directory2 + "/file2*").size());
- assertEquals(0, storage.ls(directory2 + "/file1*").size());
- assertTrue(storage.rm(directory1 + "/file1-*"));
- assertFalse(storage.rm(directory1 + "/file1-*"));
- assertEquals(0, storage.ls(directory1).size());
- assertEquals(0, storage.ls(directory1 + "/*").size());
- assertEquals(5, storage.ls(directory2).size());
- assertEquals(5, storage.ls(directory2 + "/*").size());
- assertTrue(storage.rm(directory2 + "/f*"));
- assertFalse(storage.rm(directory2 + "/file*"));
- assertEquals(0, storage.ls(directory2).size());
- assertEquals(0, storage.ls(directory2 + "*").size());
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java
deleted file mode 100644
index c2eeac3..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/FileSystemStorageCheck.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop.structure.io;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.structure.io.Storage;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class FileSystemStorageCheck extends AbstractStorageCheck {
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldSupportHeadMethods() throws Exception {
- final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
- final String inputLocation = Constants.getSearchGraphLocation(graph.configuration().getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION), storage).get();
- final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
- // TestHelper creates the directory and we need it not to exist
- deleteDirectory(outputLocation);
- super.checkHeadMethods(storage, inputLocation, outputLocation, InputOutputHelper.getInputFormat((Class) Class.forName(graph.configuration().getString(Constants.GREMLIN_HADOOP_GRAPH_WRITER))), SequenceFileInputFormat.class);
- }
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldSupportRemoveAndListMethods() throws Exception {
- final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
- final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
- super.checkRemoveAndListMethods(storage, outputLocation);
- }
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldSupportCopyMethods() throws Exception {
- final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
- final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
- final String newOutputLocation = TestHelper.makeTestDataDirectory(FileSystemStorageCheck.class, "new-location-for-copy");
- // TestHelper creates the directory and we need it not to exist
- deleteDirectory(newOutputLocation);
- super.checkCopyMethods(storage, outputLocation, newOutputLocation, InputOutputHelper.getInputFormat((Class) Class.forName(graph.configuration().getString(Constants.GREMLIN_HADOOP_GRAPH_WRITER))), SequenceFileInputFormat.class);
-
- }
-
- @Test
- @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
- public void shouldNotHaveResidualDataInStorage() throws Exception {
- final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
- final String outputLocation = graph.configuration().getString(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
- super.checkResidualDataInStorage(storage, outputLocation);
- }
-
- @Test
- public void shouldSupportDirectoryFileDistinction() throws Exception {
- final Storage storage = FileSystemStorage.open(ConfUtil.makeHadoopConfiguration(graph.configuration()));
- final String directory1 = TestHelper.makeTestDataDirectory(FileSystemStorageCheck.class, "directory1");
- final String directory2 = TestHelper.makeTestDataDirectory(FileSystemStorageCheck.class, "directory2");
- for (int i = 0; i < 10; i++) {
- new File(directory1 + "/" + "file1-" + i + ".txt.bz").createNewFile();
- }
- for (int i = 0; i < 5; i++) {
- new File(directory2 + "/" + "file2-" + i + ".txt.bz").createNewFile();
- }
- super.checkFileDirectoryDistinction(storage, directory1, directory2);
- deleteDirectory(directory1);
- deleteDirectory(directory2);
- }
-
- private static void deleteDirectory(final String location) throws IOException {
- // TestHelper creates the directory and we need it not to exist
- assertTrue(new File(location).isDirectory());
- assertTrue(new File(location).exists());
- FileUtils.deleteDirectory(new File(location));
- assertFalse(new File(location).exists());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyIoRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyIoRegistry.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyIoRegistry.java
deleted file mode 100644
index 26d87a9..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyIoRegistry.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo;
-
-import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ToyIoRegistry extends AbstractIoRegistry {
-
- private static final ToyIoRegistry INSTANCE = new ToyIoRegistry();
-
- private ToyIoRegistry() {
- super.register(GryoIo.class, ToyPoint.class, new ToyPoint.ToyPointSerializer());
- super.register(GryoIo.class, ToyTriangle.class, new ToyTriangle.ToyTriangleSerializer());
- super.register(GraphSONIo.class, null, new ToyModule());
- }
-
- public static class ToyModule extends TinkerPopJacksonModule {
- public ToyModule() {
- super("toy");
- addSerializer(ToyPoint.class, new ToyPoint.ToyPointJacksonSerializer());
- addDeserializer(ToyPoint.class, new ToyPoint.ToyPointJacksonDeSerializer());
- addSerializer(ToyTriangle.class, new ToyTriangle.ToyTriangleJacksonSerializer());
- addDeserializer(ToyTriangle.class, new ToyTriangle.ToyTriangleJacksonDeSerializer());
- }
-
-
- @Override
- public Map<Class, String> getTypeDefinitions() {
- return new HashMap<Class, String>() {{
- put(ToyPoint.class, "ToyPoint");
- put(ToyTriangle.class, "ToyTriangle");
- }};
- }
-
- @Override
- public String getTypeNamespace() {
- return "toy";
- }
- }
-
- public static ToyIoRegistry instance() {
- return INSTANCE;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyPoint.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyPoint.java
deleted file mode 100644
index d0a5ca3..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyPoint.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo;
-
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.AbstractObjectDeserializer;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
-import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException;
-import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
-import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider;
-import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
-import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ToyPoint {
-
- private final int x;
- private final int y;
-
- public ToyPoint(final int x, final int y) {
- this.x = x;
- this.y = y;
- }
-
- public int getX() {
- return this.x;
- }
-
- public int getY() {
- return this.y;
- }
-
- public int hashCode() {
- return this.x + this.y;
- }
-
- public boolean equals(final Object other) {
- return other instanceof ToyPoint && ((ToyPoint) other).x == this.x && ((ToyPoint) other).y == this.y;
- }
-
- @Override
- public String toString() {
- return "[" + this.x + "," + this.y + "]";
- }
-
- public static class ToyPointSerializer implements SerializerShim<ToyPoint> {
- @Override
- public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ToyPoint toyPoint) {
- output.writeInt(toyPoint.x);
- output.writeInt(toyPoint.y);
- }
-
- @Override
- public <I extends InputShim> ToyPoint read(final KryoShim<I, ?> kryo, final I input, final Class<ToyPoint> toyPointClass) {
- return new ToyPoint(input.readInt(), input.readInt());
- }
- }
-
- public static class ToyPointJacksonSerializer extends StdScalarSerializer<ToyPoint> {
-
- public ToyPointJacksonSerializer() {
- super(ToyPoint.class);
- }
-
- @Override
- public void serialize(final ToyPoint toyPoint, final JsonGenerator jsonGenerator,
- final SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectField("x", toyPoint.x);
- jsonGenerator.writeObjectField("y", toyPoint.y);
- jsonGenerator.writeEndObject();
- }
- }
-
- public static class ToyPointJacksonDeSerializer extends AbstractObjectDeserializer<ToyPoint> {
-
- public ToyPointJacksonDeSerializer() {
- super(ToyPoint.class);
- }
-
- @Override
- public ToyPoint createObject(final Map<String, Object> map) {
- return new ToyPoint((int) map.get("x"), (int) map.get("y"));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyTriangle.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyTriangle.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyTriangle.java
deleted file mode 100644
index 280cd01..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/gryo/ToyTriangle.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo;
-
-import org.apache.tinkerpop.gremlin.structure.io.graphson.AbstractObjectDeserializer;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.SerializerShim;
-import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException;
-import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator;
-import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider;
-import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer;
-import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ToyTriangle {
-
- private final int x;
- private final int y;
- private final int z;
-
- public ToyTriangle(final int x, final int y, final int z) {
- this.x = x;
- this.y = y;
- this.z = z;
- }
-
- public int getX() {
- return this.x;
- }
-
- public int getY() {
- return this.y;
- }
-
- public int getZ() {
- return this.z;
- }
-
- public int hashCode() {
- return this.x + this.y + this.z;
- }
-
- public boolean equals(final Object other) {
- return other instanceof ToyTriangle && ((ToyTriangle) other).x == this.x && ((ToyTriangle) other).y == this.y && ((ToyTriangle) other).z == this.z;
- }
-
- @Override
- public String toString() {
- return "[" + this.x + "," + this.y + "," + this.z + "]";
- }
-
- public static class ToyTriangleSerializer implements SerializerShim<ToyTriangle> {
- @Override
- public <O extends OutputShim> void write(final KryoShim<?, O> kryo, final O output, final ToyTriangle toyTriangle) {
- output.writeInt(toyTriangle.x);
- output.writeInt(toyTriangle.y);
- output.writeInt(toyTriangle.z);
- }
-
- @Override
- public <I extends InputShim> ToyTriangle read(final KryoShim<I, ?> kryo, final I input, final Class<ToyTriangle> toyTriangleClass) {
- return new ToyTriangle(input.readInt(), input.readInt(), input.readInt());
- }
- }
-
-
- public static class ToyTriangleJacksonSerializer extends StdScalarSerializer<ToyTriangle> {
-
- public ToyTriangleJacksonSerializer() {
- super(ToyTriangle.class);
- }
-
- @Override
- public void serialize(final ToyTriangle toyTriangle, final JsonGenerator jsonGenerator,
- final SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
- jsonGenerator.writeStartObject();
- jsonGenerator.writeObjectField("x", toyTriangle.x);
- jsonGenerator.writeObjectField("y", toyTriangle.y);
- jsonGenerator.writeObjectField("z", toyTriangle.z);
- jsonGenerator.writeEndObject();
- }
- }
-
- public static class ToyTriangleJacksonDeSerializer extends AbstractObjectDeserializer<ToyTriangle> {
-
- public ToyTriangleJacksonDeSerializer() {
- super(ToyTriangle.class);
- }
-
- @Override
- public ToyTriangle createObject(final Map<String, Object> map) {
- return new ToyTriangle((int) map.get("x"), (int) map.get("y"), (int) map.get("z"));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 12680c9..ad3bf43 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -307,63 +307,6 @@
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
- <artifactId>hadoop-gremlin</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-net</groupId>
- <artifactId>commons-net</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.xerial.snappy</groupId>
- <artifactId>snappy-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.tinkerpop</groupId>
<artifactId>tinkergraph-gremlin</artifactId>
<version>${project.version}</version>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7eec250a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java
new file mode 100644
index 0000000..2bce560
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.hadoop;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorageCheck;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopGremlinSuite extends AbstractGremlinSuite {
+ public HadoopGremlinSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
+ super(klass, builder, new Class<?>[]{FileSystemStorageCheck.class}, new Class<?>[]{FileSystemStorageCheck.class}, true, TraversalEngine.Type.COMPUTER);
+ }
+}
[3/3] tinkerpop git commit: Merge branch 'TINKERPOP-1410'
Posted by sp...@apache.org.
Merge branch 'TINKERPOP-1410'
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/4e5c30e2
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/4e5c30e2
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/4e5c30e2
Branch: refs/heads/master
Commit: 4e5c30e20efc4b7f0f3107097a810632f4c81776
Parents: 108b236 7eec250
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Mon Apr 16 07:56:13 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Mon Apr 16 07:56:13 2018 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/upgrade/release-3.4.x.asciidoc | 22 +++
.../gremlin/AbstractFileGraphProvider.java | 110 +++++++++++
.../structure/io/gryo/ToyIoRegistry.java | 69 +++++++
.../gremlin/structure/io/gryo/ToyPoint.java | 109 +++++++++++
.../gremlin/structure/io/gryo/ToyTriangle.java | 119 ++++++++++++
hadoop-gremlin/pom.xml | 8 -
.../gremlin/hadoop/HadoopGraphProvider.java | 85 +--------
.../gremlin/hadoop/HadoopGremlinSuite.java | 35 ----
.../structure/io/AbstractIoRegistryCheck.java | 137 -------------
.../structure/io/AbstractStorageCheck.java | 190 -------------------
.../structure/io/FileSystemStorageCheck.java | 104 ----------
.../hadoop/structure/io/gryo/ToyIoRegistry.java | 70 -------
.../hadoop/structure/io/gryo/ToyPoint.java | 113 -----------
.../hadoop/structure/io/gryo/ToyTriangle.java | 120 ------------
spark-gremlin/pom.xml | 57 ------
.../gremlin/hadoop/HadoopGremlinSuite.java | 35 ++++
.../structure/io/AbstractIoRegistryCheck.java | 136 +++++++++++++
.../structure/io/AbstractStorageCheck.java | 190 +++++++++++++++++++
.../structure/io/FileSystemStorageCheck.java | 104 ++++++++++
.../computer/SparkHadoopGraphProvider.java | 59 +++++-
.../gremlin/spark/util/SugarTestHelper.java | 4 +-
22 files changed, 955 insertions(+), 922 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4e5c30e2/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 234d456,c577eea..d63e91a
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -25,8 -25,8 +25,9 @@@ NEED AND IMAG
This release also includes changes from <<release-3-3-2, 3.3.2>>.
+* `min()` and `max()` now support all types implementing `Comparable`.
* Change the `toString()` of `Path` to be standardized as other graph elements are.
+ * `hadoop-gremlin` no longer generates a test artifact.
* Fixed a bug in `ReducingBarrierStep`, that returned the provided seed value despite no elements being available.
* Changed the order of `select()` scopes. The order is now: maps, side-effects, paths.
* Removed support for Giraph.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4e5c30e2/docs/src/upgrade/release-3.4.x.asciidoc
----------------------------------------------------------------------
diff --cc docs/src/upgrade/release-3.4.x.asciidoc
index 57a6fdb,bdf443e..f29df13
--- a/docs/src/upgrade/release-3.4.x.asciidoc
+++ b/docs/src/upgrade/release-3.4.x.asciidoc
@@@ -155,3 -122,25 +155,25 @@@ gremlin> g.V(1)
----
See: link:https://issues.apache.org/jira/browse/TINKERPOP-1522[TINKERPOP-1522]
+
+ === Upgrading for Providers
+
+ ==== Graph Providers
+
+ ===== Removed hadoop-gremlin Test Artifact
+
+ The `hadoop-gremlin` module no longer generates a test jar that can be used as a test dependency in other modules.
+ Generally speaking, that approach tends to be a bad practice and can cause build problems with Maven that aren't always
+ obvious to troubleshoot. With the removal of `giraph-gremlin` for 3.4.0, it seemed even less useful to have this
+ test artifact present. All tests are still present. The follow provides a basic summary of how this refactoring
+ occurred:
+
+ * A new `AbstractFileGraphProvider` was created in `gremlin-test` which provided a lot of the features that
+ `HadoopGraphProvider` was exposing. Both `HadoopGraphProvider` and `SparkHadoopGraphProvider` extend from that class
+ now.
+ * `ToyIoRegistry` and related classes were moved to `gremlin-test`.
+ * The various tests that validated capabilities of `Storage` have been moved to `spark-gremlin` and are part of those
+ tests now. Obviously, that makes those tests specific to Spark testing now. If that location creates a problem for some
+ reason, that decision can be revisited at some point.
+
-See: link:https://issues.apache.org/jira/browse/TINKERPOP-1410[TINKERPOP-1410]
++See: link:https://issues.apache.org/jira/browse/TINKERPOP-1410[TINKERPOP-1410]