You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/09/08 23:45:20 UTC
incubator-tinkerpop git commit: More organization of Hadoop split.
Note that Giraph OPT_OUTs are no longer needed. Suspect some jar conflict was
causing the hard thread stop previously. Now that Giraph is isolated,
tests just pass.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/hadoop_split efe085d06 -> ffb42a1fd
More organization of Hadoop split. Note that Giraph OPT_OUTs are no longer needed. Suspect some jar conflict was causing the hard thread stop previously. Now that Giraph is isolated, tests just pass.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ffb42a1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ffb42a1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ffb42a1f
Branch: refs/heads/hadoop_split
Commit: ffb42a1fd410fdbcc664a5e73178bf692449db1f
Parents: efe085d
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 8 15:45:45 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 8 15:45:45 2015 -0600
----------------------------------------------------------------------
.../groovy/plugin/GiraphGremlinPlugin.java | 72 +++++++++
.../groovy/plugin/HadoopGremlinPluginTest.java | 162 +++++++++++++++++++
.../giraph/groovy/plugin/HadoopPluginSuite.java | 36 +++++
.../giraph/process/HadoopGraphProvider.java | 161 ------------------
.../computer/HadoopGiraphGraphProvider.java | 136 +++++++++++++++-
.../GiraphHadoopGremlinPluginIntegrateTest.java | 3 +-
.../gremlin/hadoop/structure/HadoopGraph.java | 20 +--
.../spark/groovy/plugin/SparkGremlinPlugin.java | 2 +-
.../groovy/plugin/HadoopGremlinPluginTest.java | 162 +++++++++++++++++++
.../spark/groovy/plugin/HadoopPluginSuite.java | 36 +++++
.../spark/process/HadoopGraphProvider.java | 151 -----------------
.../computer/HadoopSparkGraphProvider.java | 128 ++++++++++++++-
.../groovy/SparkHadoopGremlinPluginTest.java | 3 +-
13 files changed, 733 insertions(+), 339 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
new file mode 100644
index 0000000..f50b686
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.giraph.groovy.plugin;
+
+import org.apache.tinkerpop.giraph.process.computer.GiraphGraphComputer;
+import org.apache.tinkerpop.gremlin.groovy.plugin.AbstractGremlinPlugin;
+import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphGremlinPlugin extends AbstractGremlinPlugin {
+
+ protected static String NAME = "tinkerpop.spark";
+
+ protected static final Set<String> IMPORTS = new HashSet<String>() {{
+ add(IMPORT_SPACE + GiraphGraphComputer.class.getPackage().getName() + DOT_STAR);
+ }};
+
+ public GiraphGremlinPlugin() {
+ super(true);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public void afterPluginTo(final PluginAcceptor pluginAcceptor) throws PluginInitializationException, IllegalEnvironmentException {
+ pluginAcceptor.addImports(IMPORTS);
+ try {
+ pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", GiraphGraphComputer.class.getName()));
+ } catch (final Exception e) {
+ throw new PluginInitializationException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public boolean requireRestart() {
+ return true;
+ }
+
+ @Override
+ public Optional<RemoteAcceptor> remoteAcceptor() {
+ return Optional.empty();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java
new file mode 100644
index 0000000..d9fd95e
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.HadoopGiraphGraphProvider;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.util.TestableConsolePluginAcceptor;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPlugin;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopRemoteAcceptor;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.sideEffect.mapreduce.TraverserMapReduce;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+
+public class HadoopGremlinPluginTest extends AbstractGremlinTest {
+
+ @Before
+ public void setupTest() {
+ try {
+ this.console = new TestableConsolePluginAcceptor();
+ final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin();
+ plugin.pluginTo(this.console);
+ this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ ///////////////////
+
+ private HadoopRemoteAcceptor remote;
+ private TestableConsolePluginAcceptor console;
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ //
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()"));
+ assertEquals(6L, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteSugarTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph"));
+ //
+ this.remote.configure(Arrays.asList("useSugar", "true"));
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum"));
+ assertEquals(28.0d, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportHDFSMethods() throws Exception {
+ ////////////////
+ List<String> ls = (List<String>) this.console.eval("hdfs.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ls = (List<String>) this.console.eval("local.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ////////////////
+ this.console.eval("hdfs.copyFromLocal('" + HadoopGiraphGraphProvider.PATHS.get("tinkerpop-classic.txt") + "', 'target/tinkerpop-classic.txt')");
+ assertTrue((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ List<String> head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt')"));
+ assertEquals(6, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 6);
+ }
+ head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt',3)"));
+ assertEquals(3, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 3);
+ }
+ ////////////////
+ this.console.eval("hdfs.rm('target/tinkerpop-classic.txt')");
+ assertFalse((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<Vertex, String> traversal = (Traversal<Vertex, String>) this.remote.submit(Arrays.asList("g.V().hasLabel('person').group('m').by('age').by('name').out('knows').out('created').values('name')"));
+ AbstractGremlinProcessTest.checkResults(Arrays.asList("ripple", "lop"), traversal);
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/m')"));
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "')"));
+ final List<KeyValue<Integer, BulkSet<String>>> mList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/m',ObjectWritable)"));
+ assertEquals(4, mList.size());
+ mList.forEach(keyValue -> {
+ if (keyValue.getKey().equals(29))
+ assertEquals(1l, keyValue.getValue().get("marko"));
+ else if (keyValue.getKey().equals(35))
+ assertEquals(1l, keyValue.getValue().get("peter"));
+ else if (keyValue.getKey().equals(32))
+ assertEquals(1l, keyValue.getValue().get("josh"));
+ else if (keyValue.getKey().equals(27))
+ assertEquals(1l, keyValue.getValue().get("vadas"));
+ else
+ throw new IllegalStateException("The provided key/value is unknown: " + keyValue);
+ });
+ final List<KeyValue<MapReduce.NullObject, Traverser<String>>> traversersList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "',ObjectWritable)"));
+ assertEquals(2, traversersList.size());
+ traversersList.forEach(keyValue -> {
+ assertEquals(MapReduce.NullObject.instance(), keyValue.getKey());
+ final String name = keyValue.getValue().get();
+ assertTrue(name.equals("ripple") || name.equals("lop"));
+ });
+ ////////////////
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java
new file mode 100644
index 0000000..d7d9bc4
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopPluginSuite extends AbstractGremlinSuite {
+
+ public HadoopPluginSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
+ super(klass, builder, new Class<?>[]{HadoopGremlinPluginTest.class}, new Class<?>[]{HadoopGremlinPluginTest.class}, true, TraversalEngine.Type.COMPUTER);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
deleted file mode 100644
index ae0ae10..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.giraph.process;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class HadoopGraphProvider extends AbstractGraphProvider {
-
- private static final Random RANDOM = new Random();
- private boolean graphSONInput = false;
-
- public static Map<String, String> PATHS = new HashMap<>();
- private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
- add(HadoopEdge.class);
- add(HadoopElement.class);
- add(HadoopGraph.class);
- add(HadoopProperty.class);
- add(HadoopVertex.class);
- add(HadoopVertexProperty.class);
- }};
-
- static {
- try {
- final List<String> kryoResources = Arrays.asList(
- "tinkerpop-modern.kryo",
- "grateful-dead.kryo",
- "tinkerpop-classic.kryo",
- "tinkerpop-crew.kryo");
- for (final String fileName : kryoResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> graphsonResources = Arrays.asList(
- "tinkerpop-modern.json",
- "grateful-dead.json",
- "tinkerpop-classic.json",
- "tinkerpop-crew.json");
- for (final String fileName : graphsonResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> scriptResources = Arrays.asList(
- "tinkerpop-classic.txt",
- "script-input.groovy",
- "script-output.groovy",
- "grateful-dead.txt",
- "script-input-grateful-dead.groovy",
- "script-output-grateful-dead.groovy");
- for (final String fileName : scriptResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- this.graphSONInput = RANDOM.nextBoolean();
- return new HashMap<String, Object>() {{
- put(Graph.GRAPH, HadoopGraph.class.getName());
- put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
- put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- /// giraph configuration
- put(GiraphConstants.MIN_WORKERS, 1);
- put(GiraphConstants.MAX_WORKERS, 1);
- put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
- put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
- put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
- put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
- put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
- put("mapred.reduce.tasks", 4);
- put("giraph.vertexOutputFormatThreadSafe", false);
- put("giraph.numOutputThreads", 3);
- }};
- }
-
- @Override
- public void clear(final Graph graph, final Configuration configuration) throws Exception {
- if (graph != null)
- graph.close();
- }
-
- @Override
- public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
- if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
- }
-
- @Override
- public Set<Class> getImplementations() {
- return IMPLEMENTATION;
- }
-
- public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
- final String type = this.graphSONInput ? "json" : "kryo";
-
- if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
- } else {
- throw new RuntimeException("Could not load graph with " + graphData);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
index 8b124e5..78ea1bb 100644
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
@@ -18,20 +18,152 @@
*/
package org.apache.tinkerpop.gremlin.giraph.process.computer;
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.conf.GiraphConstants;
import org.apache.tinkerpop.giraph.process.computer.GiraphGraphComputer;
+import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.GraphProvider;
-import org.apache.tinkerpop.gremlin.giraph.process.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@GraphProvider.Descriptor(computer = GiraphGraphComputer.class)
-public final class HadoopGiraphGraphProvider extends HadoopGraphProvider {
+public final class HadoopGiraphGraphProvider extends AbstractGraphProvider {
+
+ private static final Random RANDOM = new Random();
+ private boolean graphSONInput = false;
+
+ public static Map<String, String> PATHS = new HashMap<>();
+ private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
+ add(HadoopEdge.class);
+ add(HadoopElement.class);
+ add(HadoopGraph.class);
+ add(HadoopProperty.class);
+ add(HadoopVertex.class);
+ add(HadoopVertexProperty.class);
+ }};
+
+ static {
+ try {
+ final List<String> kryoResources = Arrays.asList(
+ "tinkerpop-modern.kryo",
+ "grateful-dead.kryo",
+ "tinkerpop-classic.kryo",
+ "tinkerpop-crew.kryo");
+ for (final String fileName : kryoResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> graphsonResources = Arrays.asList(
+ "tinkerpop-modern.json",
+ "grateful-dead.json",
+ "tinkerpop-classic.json",
+ "tinkerpop-crew.json");
+ for (final String fileName : graphsonResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> scriptResources = Arrays.asList(
+ "tinkerpop-classic.txt",
+ "script-input.groovy",
+ "script-output.groovy",
+ "grateful-dead.txt",
+ "script-input-grateful-dead.groovy",
+ "script-output-grateful-dead.groovy");
+ for (final String fileName : scriptResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ this.graphSONInput = RANDOM.nextBoolean();
+ return new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ /// giraph configuration
+ put(GiraphConstants.MIN_WORKERS, 1);
+ put(GiraphConstants.MAX_WORKERS, 1);
+ put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
+ put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
+ put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
+ put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
+ put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
+ put("mapred.reduce.tasks", 4);
+ put("giraph.vertexOutputFormatThreadSafe", false);
+ put("giraph.numOutputThreads", 3);
+ }};
+ }
+ @Override
public GraphTraversalSource traversal(final Graph graph) {
return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(GiraphGraphComputer.class)).create(graph);
}
+
+ @Override
+ public void clear(final Graph graph, final Configuration configuration) throws Exception {
+ if (graph != null)
+ graph.close();
+ }
+
+ @Override
+ public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
+ if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
+ }
+
+ @Override
+ public Set<Class> getImplementations() {
+ return IMPLEMENTATION;
+ }
+
+ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
+ final String type = this.graphSONInput ? "json" : "kryo";
+
+ if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
+ } else {
+ throw new RuntimeException("Could not load graph with " + graphData);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
index fd87d2e..28b3692 100644
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
@@ -19,6 +19,7 @@
package org.apache.tinkerpop.gremlin.giraph.process.computer.groovy;
import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.giraph.groovy.plugin.HadoopPluginSuite;
import org.apache.tinkerpop.gremlin.giraph.process.computer.HadoopGiraphGraphProvider;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.junit.runner.RunWith;
@@ -26,7 +27,7 @@ import org.junit.runner.RunWith;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-//@RunWith(HadoopPluginSuite.class)
+@RunWith(HadoopPluginSuite.class)
@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
public class GiraphHadoopGremlinPluginIntegrateTest {
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index ae4f910..6293b31 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -78,7 +78,7 @@ import java.util.stream.Stream;
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
// computers = {GiraphGraphComputer.class})
@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@Graph.OptOut(
@@ -137,21 +137,6 @@ import java.util.stream.Stream;
method = "g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldNotAllowNullMemoryKeys",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldNotAllowSettingUndeclaredMemoryKeys",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldHaveConsistentMemoryVertexPropertiesAndExceptions",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
-@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileTest$Traversals",
method = "g_V_out_out_profile_grateful",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@@ -188,9 +173,6 @@ public final class HadoopGraph implements Graph {
@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) {
- //if (graphComputerClass.equals(GiraphGraphComputer.class))
- // return (C) new GiraphGraphComputer(this);
- //else {
try {
return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
} catch (final Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
index 51a88b3..f5300fe 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
@@ -33,7 +33,7 @@ import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class SparkGremlinPlugin extends AbstractGremlinPlugin {
+public final class SparkGremlinPlugin extends AbstractGremlinPlugin {
protected static String NAME = "tinkerpop.spark";
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java
new file mode 100644
index 0000000..5931418
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.util.TestableConsolePluginAcceptor;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPlugin;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopRemoteAcceptor;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.sideEffect.mapreduce.TraverserMapReduce;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
+import org.apache.tinkerpop.gremlin.spark.process.computer.HadoopSparkGraphProvider;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+
+public class HadoopGremlinPluginTest extends AbstractGremlinTest {
+
+ @Before
+ public void setupTest() {
+ try {
+ this.console = new TestableConsolePluginAcceptor();
+ final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin();
+ plugin.pluginTo(this.console);
+ this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ ///////////////////
+
+ private HadoopRemoteAcceptor remote;
+ private TestableConsolePluginAcceptor console;
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ //
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()"));
+ assertEquals(6L, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteSugarTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph"));
+ //
+ this.remote.configure(Arrays.asList("useSugar", "true"));
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum"));
+ assertEquals(28.0d, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportHDFSMethods() throws Exception {
+ ////////////////
+ List<String> ls = (List<String>) this.console.eval("hdfs.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ls = (List<String>) this.console.eval("local.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ////////////////
+ this.console.eval("hdfs.copyFromLocal('" + HadoopSparkGraphProvider.PATHS.get("tinkerpop-classic.txt") + "', 'target/tinkerpop-classic.txt')");
+ assertTrue((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ List<String> head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt')"));
+ assertEquals(6, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 6);
+ }
+ head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt',3)"));
+ assertEquals(3, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 3);
+ }
+ ////////////////
+ this.console.eval("hdfs.rm('target/tinkerpop-classic.txt')");
+ assertFalse((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<Vertex, String> traversal = (Traversal<Vertex, String>) this.remote.submit(Arrays.asList("g.V().hasLabel('person').group('m').by('age').by('name').out('knows').out('created').values('name')"));
+ AbstractGremlinProcessTest.checkResults(Arrays.asList("ripple", "lop"), traversal);
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/m')"));
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "')"));
+ final List<KeyValue<Integer, BulkSet<String>>> mList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/m',ObjectWritable)"));
+ assertEquals(4, mList.size());
+ mList.forEach(keyValue -> {
+ if (keyValue.getKey().equals(29))
+ assertEquals(1l, keyValue.getValue().get("marko"));
+ else if (keyValue.getKey().equals(35))
+ assertEquals(1l, keyValue.getValue().get("peter"));
+ else if (keyValue.getKey().equals(32))
+ assertEquals(1l, keyValue.getValue().get("josh"));
+ else if (keyValue.getKey().equals(27))
+ assertEquals(1l, keyValue.getValue().get("vadas"));
+ else
+ throw new IllegalStateException("The provided key/value is unknown: " + keyValue);
+ });
+ final List<KeyValue<MapReduce.NullObject, Traverser<String>>> traversersList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "',ObjectWritable)"));
+ assertEquals(2, traversersList.size());
+ traversersList.forEach(keyValue -> {
+ assertEquals(MapReduce.NullObject.instance(), keyValue.getKey());
+ final String name = keyValue.getValue().get();
+ assertTrue(name.equals("ripple") || name.equals("lop"));
+ });
+ ////////////////
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java
new file mode 100644
index 0000000..bcce37c
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopPluginSuite extends AbstractGremlinSuite {
+
+ public HadoopPluginSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
+ super(klass, builder, new Class<?>[]{HadoopGremlinPluginTest.class}, new Class<?>[]{HadoopGremlinPluginTest.class}, true, TraversalEngine.Type.COMPUTER);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
deleted file mode 100644
index cdc4a0b..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class HadoopGraphProvider extends AbstractGraphProvider {
-
- private static final Random RANDOM = new Random();
- private boolean graphSONInput = false;
-
- public static Map<String, String> PATHS = new HashMap<>();
- private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
- add(HadoopEdge.class);
- add(HadoopElement.class);
- add(HadoopGraph.class);
- add(HadoopProperty.class);
- add(HadoopVertex.class);
- add(HadoopVertexProperty.class);
- }};
-
- static {
- try {
- final List<String> kryoResources = Arrays.asList(
- "tinkerpop-modern.kryo",
- "grateful-dead.kryo",
- "tinkerpop-classic.kryo",
- "tinkerpop-crew.kryo");
- for (final String fileName : kryoResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> graphsonResources = Arrays.asList(
- "tinkerpop-modern.json",
- "grateful-dead.json",
- "tinkerpop-classic.json",
- "tinkerpop-crew.json");
- for (final String fileName : graphsonResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> scriptResources = Arrays.asList(
- "tinkerpop-classic.txt",
- "script-input.groovy",
- "script-output.groovy",
- "grateful-dead.txt",
- "script-input-grateful-dead.groovy",
- "script-output-grateful-dead.groovy");
- for (final String fileName : scriptResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- this.graphSONInput = RANDOM.nextBoolean();
- return new HashMap<String, Object>() {{
- put(Graph.GRAPH, HadoopGraph.class.getName());
- put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
- put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- put("mapred.reduce.tasks", 4);
- /// spark configuration
- put("spark.master", "local[4]");
- put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- // put("spark.kryo.registrationRequired",true);
- }};
- }
-
- @Override
- public void clear(final Graph graph, final Configuration configuration) throws Exception {
- if (graph != null)
- graph.close();
- }
-
- @Override
- public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
- if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
- }
-
- @Override
- public Set<Class> getImplementations() {
- return IMPLEMENTATION;
- }
-
- public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
- final String type = this.graphSONInput ? "json" : "kryo";
-
- if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
- } else {
- throw new RuntimeException("Could not load graph with " + graphData);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
index c0c5bba..21b14be 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
@@ -18,19 +18,141 @@
*/
package org.apache.tinkerpop.gremlin.spark.process.computer;
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.GraphProvider;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.spark.process.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
-public final class HadoopSparkGraphProvider extends HadoopGraphProvider {
+public final class HadoopSparkGraphProvider extends AbstractGraphProvider {
+
+ private static final Random RANDOM = new Random();
+ private boolean graphSONInput = false;
+
+ public static Map<String, String> PATHS = new HashMap<>();
+ private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
+ add(HadoopEdge.class);
+ add(HadoopElement.class);
+ add(HadoopGraph.class);
+ add(HadoopProperty.class);
+ add(HadoopVertex.class);
+ add(HadoopVertexProperty.class);
+ }};
+
+ static {
+ try {
+ final List<String> kryoResources = Arrays.asList(
+ "tinkerpop-modern.kryo",
+ "grateful-dead.kryo",
+ "tinkerpop-classic.kryo",
+ "tinkerpop-crew.kryo");
+ for (final String fileName : kryoResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> graphsonResources = Arrays.asList(
+ "tinkerpop-modern.json",
+ "grateful-dead.json",
+ "tinkerpop-classic.json",
+ "tinkerpop-crew.json");
+ for (final String fileName : graphsonResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> scriptResources = Arrays.asList(
+ "tinkerpop-classic.txt",
+ "script-input.groovy",
+ "script-output.groovy",
+ "grateful-dead.txt",
+ "script-input-grateful-dead.groovy",
+ "script-output-grateful-dead.groovy");
+ for (final String fileName : scriptResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ this.graphSONInput = RANDOM.nextBoolean();
+ return new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ put("mapred.reduce.tasks", 4);
+ /// spark configuration
+ put("spark.master", "local[4]");
+ put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ // put("spark.kryo.registrationRequired",true);
+ }};
+ }
+ @Override
public GraphTraversalSource traversal(final Graph graph) {
return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)).create(graph);
}
-}
+
+ @Override
+ public void clear(final Graph graph, final Configuration configuration) throws Exception {
+ if (graph != null)
+ graph.close();
+ }
+
+ @Override
+ public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
+ if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
+ }
+
+ @Override
+ public Set<Class> getImplementations() {
+ return IMPLEMENTATION;
+ }
+
+ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
+ final String type = this.graphSONInput ? "json" : "kryo";
+
+ if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
+ } else {
+ throw new RuntimeException("Could not load graph with " + graphData);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
index ddfe2d9..f269c4a 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
@@ -20,13 +20,14 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.groovy;
import org.apache.tinkerpop.gremlin.GraphProviderClass;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.spark.groovy.plugin.HadoopPluginSuite;
import org.apache.tinkerpop.gremlin.spark.process.computer.HadoopSparkGraphProvider;
import org.junit.runner.RunWith;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-//@RunWith(HadoopPluginSuite.class)
+@RunWith(HadoopPluginSuite.class)
@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
public class SparkHadoopGremlinPluginTest {
}