You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2021/09/07 16:06:20 UTC
[tinkerpop] 07/08: TINKERPOP-2601 Set up gherkin for HadoopGraph
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch TINKERPOP-2601
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit e75f383131869925650fc6e27ebf21ac06a264cf
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 18 10:50:39 2021 -0400
TINKERPOP-2601 Set up gherkin for HadoopGraph
---
.../gremlin/AbstractFileGraphProvider.java | 70 +--------
.../TestFiles.java} | 29 +---
hadoop-gremlin/pom.xml | 12 ++
.../gremlin/hadoop/structure/HadoopGraph.java | 5 -
.../gremlin/hadoop/HadoopGraphFeatureTest.java | 168 +++++++++++++++++++++
.../structure/io/RecordReaderWriterTest.java | 3 +-
.../io/script/ScriptRecordReaderWriterTest.java | 5 +-
.../src/test/resources/cucumber.properties | 1 +
.../spark/process/computer/LocalPropertyTest.java | 3 +-
.../optimization/SparkInterceptorStrategyTest.java | 5 +-
.../SparkSingleIterationStrategyTest.java | 3 +-
.../gremlin/spark/structure/SparkTest.java | 3 +-
.../gremlin/spark/structure/io/OutputRDDTest.java | 3 +-
.../io/PersistedInputOutputRDDIntegrateTest.java | 11 +-
14 files changed, 212 insertions(+), 109 deletions(-)
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
index cedbf38..499822f 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
@@ -19,16 +19,9 @@
package org.apache.tinkerpop.gremlin;
import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.Storage;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import java.util.Random;
/**
@@ -43,51 +36,6 @@ public abstract class AbstractFileGraphProvider extends AbstractGraphProvider {
protected boolean graphSONInput = false;
- public static Map<String, String> PATHS = new HashMap<>();
- static {
- try {
- final List<String> kryoResources = Arrays.asList(
- "tinkerpop-modern-v3d0.kryo",
- "grateful-dead-v3d0.kryo",
- "tinkerpop-classic-v3d0.kryo",
- "tinkerpop-crew-v3d0.kryo",
- "tinkerpop-sink-v3d0.kryo");
- for (final String fileName : kryoResources) {
- PATHS.put(fileName,
- Storage.toPath(TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "")));
- }
-
- final List<String> graphsonResources = Arrays.asList(
- "tinkerpop-modern-typed-v2d0.json",
- "tinkerpop-modern-v3d0.json",
- "grateful-dead-typed-v2d0.json",
- "grateful-dead-v3d0.json",
- "tinkerpop-classic-typed-v2d0.json",
- "tinkerpop-classic-v3d0.json",
- "tinkerpop-crew-typed-v2d0.json",
- "tinkerpop-crew-v3d0.json",
- "tinkerpop-sink-v3d0.json");
- for (final String fileName : graphsonResources) {
- PATHS.put(fileName,
- Storage.toPath(TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "")));
- }
-
- final List<String> scriptResources = Arrays.asList(
- "tinkerpop-classic.txt",
- "script-input.groovy",
- "script-output.groovy",
- "grateful-dead.txt",
- "script-input-grateful-dead.groovy",
- "script-output-grateful-dead.groovy");
- for (final String fileName : scriptResources) {
- PATHS.put(fileName,
- Storage.toPath(TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "")));
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
@Override
public void clear(final Graph graph, final Configuration configuration) throws Exception {
if (graph != null)
@@ -95,20 +43,6 @@ public abstract class AbstractFileGraphProvider extends AbstractGraphProvider {
}
protected String getInputLocation(final Graph g, final LoadGraphWith.GraphData graphData) {
- final String type = this.graphSONInput ? "-v3d0.json" : "-v3d0.kryo";
-
- if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL))
- return PATHS.get("grateful-dead" + type);
- else if (graphData.equals(LoadGraphWith.GraphData.MODERN))
- return PATHS.get("tinkerpop-modern" + type);
- else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC))
- return PATHS.get("tinkerpop-classic" + type);
- else if (graphData.equals(LoadGraphWith.GraphData.CREW))
- return PATHS.get("tinkerpop-crew" + type);
- else if (graphData.equals(LoadGraphWith.GraphData.SINK))
- return PATHS.get("tinkerpop-sink" + type);
- else
- throw new RuntimeException("Could not load graph with " + graphData);
+ return TestFiles.getInputLocation(graphData, graphSONInput);
}
-
}
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/features/TestFiles.java
similarity index 81%
copy from gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
copy to gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/features/TestFiles.java
index cedbf38..c923452 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/features/TestFiles.java
@@ -16,10 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.tinkerpop.gremlin;
+package org.apache.tinkerpop.gremlin.features;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.structure.io.Storage;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
@@ -31,19 +31,10 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
-/**
- * A base {@link GraphProvider} that is typically for use with Hadoop-based graphs as it enables access to the various
- * resource data files that are used in the tests.
- *
- * @author Stephen Mallette (http://stephen.genoprime.com)
- */
-public abstract class AbstractFileGraphProvider extends AbstractGraphProvider {
-
+public class TestFiles {
protected static final Random RANDOM = TestHelper.RANDOM;
- protected boolean graphSONInput = false;
-
- public static Map<String, String> PATHS = new HashMap<>();
+ public static final Map<String, String> PATHS = new HashMap<>();
static {
try {
final List<String> kryoResources = Arrays.asList(
@@ -88,14 +79,8 @@ public abstract class AbstractFileGraphProvider extends AbstractGraphProvider {
}
}
- @Override
- public void clear(final Graph graph, final Configuration configuration) throws Exception {
- if (graph != null)
- graph.close();
- }
-
- protected String getInputLocation(final Graph g, final LoadGraphWith.GraphData graphData) {
- final String type = this.graphSONInput ? "-v3d0.json" : "-v3d0.kryo";
+ public static String getInputLocation(final LoadGraphWith.GraphData graphData, final boolean useGraphSON) {
+ final String type = useGraphSON ? "-v3d0.json" : "-v3d0.kryo";
if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL))
return PATHS.get("grateful-dead" + type);
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index 19fa70e..64bb79e 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -148,6 +148,18 @@ limitations under the License.
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <version>4.2.3</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 5a2578b..a36722a 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -144,11 +144,6 @@ import java.util.stream.StreamSupport;
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
computers = {"ALL"})
@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$GreedyMatchTraversals",
- method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count",
- reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
- computers = {"ALL"})
-@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals",
method = "g_io_readXxmlX",
reason = "Hadoop-Gremlin does not support reads/writes with GraphML.")
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphFeatureTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphFeatureTest.java
new file mode 100644
index 0000000..e43b00b
--- /dev/null
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphFeatureTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.hadoop;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Stage;
+import io.cucumber.guice.CucumberModules;
+import io.cucumber.guice.GuiceFactory;
+import io.cucumber.guice.InjectorSource;
+import io.cucumber.java.Scenario;
+import io.cucumber.junit.Cucumber;
+import io.cucumber.junit.CucumberOptions;
+import org.apache.commons.configuration2.MapConfiguration;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
+import org.apache.tinkerpop.gremlin.features.World;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.javatuples.Pair;
+import org.junit.AssumptionViolatedException;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+ tags = "not @RemoteOnly and not @MultiMetaProperties",
+ glue = { "org.apache.tinkerpop.gremlin.features" },
+ objectFactory = GuiceFactory.class,
+ features = { "../gremlin-test/features" },
+ plugin = {"pretty", "junit:target/cucumber.xml"})
+public class HadoopGraphFeatureTest {
+ private static final Logger logger = LoggerFactory.getLogger(HadoopGraphFeatureTest.class);
+
+ private static final String skipReasonLength = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.";
+
+ private static final List<Pair<String, String>> skip = new ArrayList<Pair<String,String>>() {{
+ add(Pair.with("g_V_both_both_count", skipReasonLength));
+ add(Pair.with("g_V_repeatXoutX_timesX3X_count", skipReasonLength));
+ add(Pair.with("g_V_repeatXoutX_timesX8X_count", skipReasonLength));
+ add(Pair.with("g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count", skipReasonLength));
+ add(Pair.with("g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_dX_whereXc_sungBy_dX_whereXd_hasXname_GarciaXX", skipReasonLength));
+ add(Pair.with("g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_d__c_sungBy_d__d_hasXname_GarciaXX", skipReasonLength));
+ add(Pair.with("g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX", skipReasonLength));
+ add(Pair.with("g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX", skipReasonLength));
+ add(Pair.with("g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX", skipReasonLength));
+ add(Pair.with("g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX", skipReasonLength));
+ add(Pair.with("g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX", skipReasonLength));
+ add(Pair.with("g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX", skipReasonLength));
+ add(Pair.with("g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", skipReasonLength));
+ }};
+
+ public static final class ServiceModule extends AbstractModule {
+ @Override
+ protected void configure() {
+ bind(World.class).to(HadoopGraphWorld.class);
+ }
+ }
+
+ public static class HadoopGraphWorld implements World {
+
+ private static final HadoopGraph modern = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.MODERN)));
+ private static final HadoopGraph classic = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.CLASSIC)));
+ private static final HadoopGraph crew = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.CREW)));
+ private static final HadoopGraph sink = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.SINK)));
+ private static final HadoopGraph grateful = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.GRATEFUL)));
+ private static final HadoopGraph empty = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(null)));
+
+ static {
+ readIntoGraph(modern, GraphData.MODERN);
+ readIntoGraph(classic, GraphData.CLASSIC);
+ readIntoGraph(crew, GraphData.CREW);
+ readIntoGraph(sink, GraphData.SINK);
+ readIntoGraph(grateful, GraphData.GRATEFUL);
+ }
+
+ @Override
+ public GraphTraversalSource getGraphTraversalSource(final GraphData graphData) {
+ if (null == graphData)
+ throw new AssumptionViolatedException("HadoopGraph does not support graph mutations");
+ else if (graphData == GraphData.CLASSIC)
+ return classic.traversal();
+ else if (graphData == GraphData.CREW)
+ return crew.traversal();
+ else if (graphData == GraphData.MODERN)
+ return modern.traversal();
+ else if (graphData == GraphData.SINK)
+ return sink.traversal();
+ else if (graphData == GraphData.GRATEFUL)
+ return grateful.traversal();
+ else
+ throw new UnsupportedOperationException("GraphData not supported: " + graphData.name());
+ }
+
+ @Override
+ public void beforeEachScenario(final Scenario scenario) {
+ final Optional<Pair<String,String>> skipped = skip.stream().
+ filter(s -> s.getValue0().equals(scenario.getName())).findFirst();
+ if (skipped.isPresent())
+ throw new AssumptionViolatedException(skipped.get().getValue1());
+
+ cleanEmpty();
+ }
+
+ private void cleanEmpty() {
+ final GraphTraversalSource g = empty.traversal();
+ g.V().drop().iterate();
+ }
+
+ private static void readIntoGraph(final Graph graph, final GraphData graphData) {
+ ((HadoopGraph) graph).configuration().setInputLocation(TestFiles.getInputLocation(graphData, false));
+ }
+
+ private static String getWorkingDirectory() {
+ return TestHelper.makeTestDataDirectory(HadoopGraphFeatureTest.class, "graph-provider-data");
+ }
+
+ private static Map<String, Object> getBaseConfiguration(final GraphData graphData) {
+ return new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, getWorkingDirectory());
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ }};
+ }
+ }
+
+ public static final class WorldInjectorSource implements InjectorSource {
+ @Override
+ public Injector getInjector() {
+ return Guice.createInjector(Stage.PRODUCTION, CucumberModules.createScenarioModule(), new ServiceModule());
+ }
+ }
+
+}
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java
index ae93e37..48a6755 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -70,7 +71,7 @@ public abstract class RecordReaderWriterTest {
@Test
public void shouldSplitFileAndWriteProperSplits() throws Exception {
for (int numberOfSplits = 1; numberOfSplits < 10; numberOfSplits++) {
- final File testFile = new File(HadoopGraphProvider.PATHS.get(getInputFilename()));
+ final File testFile = new File(TestFiles.PATHS.get(getInputFilename()));
logger.info("Testing: {}", testFile + " (splits {}", numberOfSplits + ")");
final List<FileSplit> splits = generateFileSplits(testFile, numberOfSplits);
final Class<? extends InputFormat<NullWritable, VertexWritable>> inputFormatClass = getInputFormat();
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java
index bb0d783..554b20e 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.RecordReaderWriterTest;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
@@ -36,8 +37,8 @@ public class ScriptRecordReaderWriterTest extends RecordReaderWriterTest {
@Override
protected Configuration configure(final File outputDirectory) {
final Configuration configuration = super.configure(outputDirectory);
- configuration.set(ScriptRecordReader.SCRIPT_FILE, HadoopGraphProvider.PATHS.get("script-input-grateful-dead.groovy"));
- configuration.set(ScriptRecordWriter.SCRIPT_FILE, HadoopGraphProvider.PATHS.get("script-output-grateful-dead.groovy"));
+ configuration.set(ScriptRecordReader.SCRIPT_FILE, TestFiles.PATHS.get("script-input-grateful-dead.groovy"));
+ configuration.set(ScriptRecordWriter.SCRIPT_FILE, TestFiles.PATHS.get("script-output-grateful-dead.groovy"));
return configuration;
}
diff --git a/hadoop-gremlin/src/test/resources/cucumber.properties b/hadoop-gremlin/src/test/resources/cucumber.properties
new file mode 100644
index 0000000..66a963b
--- /dev/null
+++ b/hadoop-gremlin/src/test/resources/cucumber.properties
@@ -0,0 +1 @@
+guice.injector-source=org.apache.tinkerpop.gremlin.hadoop.HadoopGraphFeatureTest$WorldInjectorSource
\ No newline at end of file
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
index 829c405..f26d6b0 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java
@@ -26,6 +26,7 @@ import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.JavaSparkStatusTracker;
import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
@@ -56,7 +57,7 @@ public class LocalPropertyTest extends AbstractSparkTest {
configuration.setProperty("spark.master", "local[4]");
configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
index b1d58a6..869963c 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.o
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
@@ -60,7 +61,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
@Test
public void shouldHandleSideEffectsCorrectly() throws Exception {
final Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString()));
@@ -88,7 +89,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest {
@Test
public void shouldSuccessfullyEvaluateInterceptedTraversals() throws Exception {
final Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString()));
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java
index 6202f33..cbd8cd4 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.o
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep;
@@ -76,7 +77,7 @@ public class SparkSingleIterationStrategyTest extends AbstractSparkTest {
public void shouldSuccessfullyEvaluateSingleIterationTraversals() throws Exception {
final String outputLocation = TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString());
Configuration configuration = getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation);
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java
index 5aa78a4..17ef288 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java
@@ -23,6 +23,7 @@ import org.apache.commons.configuration2.BaseConfiguration;
import org.apache.commons.configuration2.Configuration;
import org.apache.spark.rdd.RDD;
import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
@@ -59,7 +60,7 @@ public class SparkTest extends AbstractSparkTest {
configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
index 7042a6c..b675615 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration2.BaseConfiguration;
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
@@ -46,7 +47,7 @@ public class OutputRDDTest extends AbstractSparkTest {
configuration.setProperty("spark.master", "local[4]");
configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, ExampleOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(this.getClass(), "shouldWriteToArbitraryRDD"));
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java
index f26efbc..ded8698 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java
@@ -22,6 +22,7 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration2.Configuration;
import org.apache.spark.storage.StorageLevel;
import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.features.TestFiles;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
@@ -56,7 +57,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest {
Spark.create("local[4]");
final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString());
final Configuration configuration = super.getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
@@ -84,7 +85,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest {
counter++;
final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString());
final Configuration configuration = super.getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_STORAGE_LEVEL, storageLevel);
@@ -112,7 +113,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest {
Spark.create("local[4]");
final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString());
final Configuration configuration = super.getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
@@ -138,7 +139,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest {
final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString());
final String rddName2 = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString());
final Configuration configuration = super.getBaseConfiguration();
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
@@ -211,7 +212,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest {
final String rddName2 = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, "testComplexChain", "graphRDD2");
final Configuration configuration = super.getBaseConfiguration();
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName);
configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);