You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/09/09 22:20:31 UTC
[01/18] incubator-tinkerpop git commit: Spark is now isolated into
spark-gremlin package. Tests are passing. A few hacks here and there just to
get things building. However,
it was pretty easy to split apart -- which is pomising.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 898615645 -> 90b28a16e
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputFormatRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputFormatRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputFormatRDD.java
new file mode 100644
index 0000000..56a1297
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputFormatRDD.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class OutputFormatRDD implements OutputRDD {
+
+ @Override
+ public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object, VertexWritable> graphRDD) {
+ final org.apache.hadoop.conf.Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(configuration);
+ final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
+ if (null != outputLocation) {
+ // map back to a <nullwritable,vertexwritable> stream for output
+ graphRDD.mapToPair(tuple -> new Tuple2<>(NullWritable.get(), tuple._2()))
+ .saveAsNewAPIHadoopFile(outputLocation + "/" + Constants.HIDDEN_G,
+ NullWritable.class,
+ VertexWritable.class,
+ (Class<OutputFormat<NullWritable, VertexWritable>>) hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, OutputFormat.class), hadoopConfiguration);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDD.java
new file mode 100644
index 0000000..2580252
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDD.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface OutputRDD {
+
+ public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object, VertexWritable> graphRDD);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
new file mode 100644
index 0000000..09e2599
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/MessagePayload.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.payload;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class MessagePayload<M> implements Payload {
+
+ private final M message;
+
+ public MessagePayload(final M message) {
+ this.message = message;
+ }
+
+ public M getMessage() {
+ return this.message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/Payload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/Payload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/Payload.java
new file mode 100644
index 0000000..5d85c61
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/Payload.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.payload;
+
+import java.io.Serializable;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface Payload extends Serializable {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
new file mode 100644
index 0000000..911fc7b
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewIncomingPayload.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.payload;
+
+import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ViewIncomingPayload<M> implements Payload {
+
+ private List<DetachedVertexProperty<Object>> view = null;
+ private final List<M> incomingMessages;
+
+
+ public ViewIncomingPayload() {
+ this.incomingMessages = null;
+ }
+
+ public ViewIncomingPayload(final MessageCombiner<M> messageCombiner) {
+ this.incomingMessages = null == messageCombiner ? new ArrayList<>() : new ArrayList<>(1);
+ }
+
+ public ViewIncomingPayload(final ViewPayload viewPayload) {
+ this.incomingMessages = null;
+ this.view = viewPayload.getView();
+ }
+
+
+ public List<DetachedVertexProperty<Object>> getView() {
+ return null == this.view ? Collections.emptyList() : this.view;
+ }
+
+
+ public List<M> getIncomingMessages() {
+ return null == this.incomingMessages ? Collections.emptyList() : this.incomingMessages;
+ }
+
+ public boolean hasView() {
+ return null != view;
+ }
+
+ ////////////////////
+
+
+ private void mergeMessage(final M message, final MessageCombiner<M> messageCombiner) {
+ if (this.incomingMessages.isEmpty() || null == messageCombiner)
+ this.incomingMessages.add(message);
+ else
+ this.incomingMessages.set(0, messageCombiner.combine(this.incomingMessages.get(0), message));
+ }
+
+ private void mergeViewIncomingPayload(final ViewIncomingPayload<M> viewIncomingPayload, final MessageCombiner<M> messageCombiner) {
+ if (this.view == null)
+ this.view = viewIncomingPayload.view;
+ else
+ this.view.addAll(viewIncomingPayload.getView());
+
+ for (final M message : viewIncomingPayload.getIncomingMessages()) {
+ this.mergeMessage(message, messageCombiner);
+ }
+ }
+
+ public void mergePayload(final Payload payload, final MessageCombiner<M> messageCombiner) {
+ if (payload instanceof ViewPayload)
+ this.view = ((ViewPayload) payload).getView();
+ else if (payload instanceof MessagePayload)
+ this.mergeMessage(((MessagePayload<M>) payload).getMessage(), messageCombiner);
+ else if (payload instanceof ViewIncomingPayload)
+ this.mergeViewIncomingPayload((ViewIncomingPayload<M>) payload, messageCombiner);
+ else
+ throw new IllegalArgumentException("The provided payload is an unsupported merge payload: " + payload);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
new file mode 100644
index 0000000..fc4aeed
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewOutgoingPayload.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.payload;
+
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
+import scala.Tuple2;
+
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ViewOutgoingPayload<M> implements Payload {
+
+ private final List<DetachedVertexProperty<Object>> view;
+ private final List<Tuple2<Object,M>> outgoingMessages;
+
+ public ViewOutgoingPayload(final List<DetachedVertexProperty<Object>> view, final List<Tuple2<Object,M>> outgoingMessages) {
+ this.view = view;
+ this.outgoingMessages = outgoingMessages;
+ }
+
+ public ViewPayload getView() {
+ return new ViewPayload(this.view);
+ }
+
+ public List<Tuple2<Object,M>> getOutgoingMessages() {
+ return this.outgoingMessages;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewPayload.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewPayload.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewPayload.java
new file mode 100644
index 0000000..dc9376d
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/payload/ViewPayload.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.payload;
+
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
+
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ViewPayload implements Payload {
+
+ private final List<DetachedVertexProperty<Object>> view;
+
+ public ViewPayload(final List<DetachedVertexProperty<Object>> view) {
+ this.view = view;
+ }
+
+ public List<DetachedVertexProperty<Object>> getView() {
+ return this.view;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
new file mode 100644
index 0000000..f085678
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopGraphProvider extends AbstractGraphProvider {
+
+ private static final Random RANDOM = new Random();
+ private boolean graphSONInput = false;
+
+ public static Map<String, String> PATHS = new HashMap<>();
+ private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
+ add(HadoopEdge.class);
+ add(HadoopElement.class);
+ add(HadoopGraph.class);
+ add(HadoopProperty.class);
+ add(HadoopVertex.class);
+ add(HadoopVertexProperty.class);
+ }};
+
+ static {
+ try {
+ final List<String> kryoResources = Arrays.asList(
+ "tinkerpop-modern.kryo",
+ "grateful-dead.kryo",
+ "tinkerpop-classic.kryo",
+ "tinkerpop-crew.kryo");
+ for (final String fileName : kryoResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> graphsonResources = Arrays.asList(
+ "tinkerpop-modern.json",
+ "grateful-dead.json",
+ "tinkerpop-classic.json",
+ "tinkerpop-crew.json");
+ for (final String fileName : graphsonResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> scriptResources = Arrays.asList(
+ "tinkerpop-classic.txt",
+ "script-input.groovy",
+ "script-output.groovy",
+ "grateful-dead.txt",
+ "script-input-grateful-dead.groovy",
+ "script-output-grateful-dead.groovy");
+ for (final String fileName : scriptResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ this.graphSONInput = RANDOM.nextBoolean();
+ return new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ /// giraph configuration
+ put(GiraphConstants.MIN_WORKERS, 1);
+ put(GiraphConstants.MAX_WORKERS, 1);
+ put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
+ put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
+ put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
+ put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
+ put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
+ put("mapred.reduce.tasks", 4);
+ //put("giraph.vertexOutputFormatThreadSafe", false);
+ //put("giraph.numOutputThreads", 3);
+
+ /// spark configuration
+ put("spark.master", "local[4]");
+ put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ // put("spark.kryo.registrationRequired",true);
+ }};
+ }
+
+ @Override
+ public void clear(final Graph graph, final Configuration configuration) throws Exception {
+ if (graph != null)
+ graph.close();
+ }
+
+ @Override
+ public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
+ if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
+ }
+
+ @Override
+ public Set<Class> getImplementations() {
+ return IMPLEMENTATION;
+ }
+
+ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
+ final String type = this.graphSONInput ? "json" : "kryo";
+
+ if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
+ } else {
+ throw new RuntimeException("Could not load graph with " + graphData);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
new file mode 100644
index 0000000..c0c5bba
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.GraphProvider;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.process.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
+public final class HadoopSparkGraphProvider extends HadoopGraphProvider {
+
+ public GraphTraversalSource traversal(final Graph graph) {
+ return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)).create(graph);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputerProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputerProcessIntegrateTest.java
new file mode 100644
index 0000000..da32c8c
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputerProcessIntegrateTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(ProcessComputerSuite.class)
+@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
+public class SparkGraphComputerProcessIntegrateTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java
new file mode 100644
index 0000000..ec1f036
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.groovy;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.process.GroovyProcessComputerSuite;
+import org.apache.tinkerpop.gremlin.spark.process.computer.HadoopSparkGraphProvider;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(GroovyProcessComputerSuite.class)
+@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
+public class SparkGraphComputerGroovyProcessIntegrateTest {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
new file mode 100644
index 0000000..ddfe2d9
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.groovy;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.spark.process.computer.HadoopSparkGraphProvider;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+//@RunWith(HadoopPluginSuite.class)
+@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
+public class SparkHadoopGremlinPluginTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleInputRDD.java
new file mode 100644
index 0000000..1fb85a1
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleInputRDD.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+import scala.Tuple2;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ExampleInputRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ final List<Vertex> list = new ArrayList<>();
+ list.add(StarGraph.open().addVertex(T.id, 1l, T.label,"person","age", 29));
+ list.add(StarGraph.open().addVertex(T.id, 2l, T.label,"person","age", 27));
+ list.add(StarGraph.open().addVertex(T.id, 4l, T.label,"person","age", 32));
+ list.add(StarGraph.open().addVertex(T.id, 6l, T.label,"person","age", 35));
+ return sparkContext.parallelize(list).mapToPair(vertex -> new Tuple2<>(vertex.id(), new VertexWritable(vertex)));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleOutputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleOutputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleOutputRDD.java
new file mode 100644
index 0000000..bb38f7f
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/ExampleOutputRDD.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class ExampleOutputRDD implements OutputRDD {
+ @Override
+ public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object, VertexWritable> graphRDD) {
+ int totalAge = 0;
+ final Iterator<VertexWritable> iterator = graphRDD.values().toLocalIterator();
+ while (iterator.hasNext()) {
+ final Vertex vertex = iterator.next().get();
+ if (vertex.label().equals("person"))
+ totalAge = totalAge + vertex.<Integer>value("age");
+ }
+ assertEquals(123, totalAge);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputOutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputOutputRDDTest.java
new file mode 100644
index 0000000..8f35b54
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputOutputRDDTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class InputOutputRDDTest {
+
+ @Test
+ public void shouldReadFromWriteToArbitraryRDD() throws Exception {
+ final Configuration configuration = new BaseConfiguration();
+ configuration.setProperty("spark.master", "local[4]");
+ configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ ////////
+ Graph graph = GraphFactory.open(configuration);
+ graph.compute(SparkGraphComputer.class)
+ .result(GraphComputer.ResultGraph.NEW)
+ .persist(GraphComputer.Persist.EDGES)
+ .program(TraversalVertexProgram.build()
+ .traversal(GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)),
+ "gremlin-groovy",
+ "g.V()").create(graph)).submit().get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDDTest.java
new file mode 100644
index 0000000..8ac9ba3
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDDTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class InputRDDTest {
+
+ @Test
+ public void shouldReadFromArbitraryRDD() {
+ final Configuration configuration = new BaseConfiguration();
+ configuration.setProperty("spark.master", "local[4]");
+ configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ ////////
+ Graph graph = GraphFactory.open(configuration);
+ assertEquals(Double.valueOf(123.0d), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
+ assertEquals(Long.valueOf(4l), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().count().next());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java
new file mode 100644
index 0000000..4ded9bc
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.spark.process.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+import org.junit.Test;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class OutputRDDTest {
+
+ @Test
+ public void shouldWriteToArbitraryRDD() throws Exception {
+ final Configuration configuration = new BaseConfiguration();
+ configuration.setProperty("spark.master", "local[4]");
+ configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, HadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName());
+ configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ ////////
+ Graph graph = GraphFactory.open(configuration);
+ graph.compute(SparkGraphComputer.class)
+ .result(GraphComputer.ResultGraph.NEW)
+ .persist(GraphComputer.Persist.EDGES)
+ .program(TraversalVertexProgram.build()
+ .traversal(GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)),
+ "gremlin-groovy",
+ "g.V()").create(graph)).submit().get();
+ }
+}
[11/18] incubator-tinkerpop git commit: Spark 1.4.1 working with TP3
(Spark's dependency scene is a mess).
Posted by ok...@apache.org.
Spark 1.4.1 working with TP3 (Spark's dependency scene is a mess).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/8de580e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/8de580e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/8de580e6
Branch: refs/heads/master
Commit: 8de580e663f9fd8c04f8c413df3cb66a42e5a577
Parents: 1303acf
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 8 12:17:32 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 8 12:17:32 2015 -0600
----------------------------------------------------------------------
.../gremlin/hadoop/HadoopGraphProvider.java | 19 --------
spark-gremlin/pom.xml | 40 ++++++++++++++--
spark-gremlin/src/assembly/hadoop-job.xml | 39 +++++++++++++++
spark-gremlin/src/assembly/standalone.xml | 48 +++++++++++++++++++
.../spark/groovy/plugin/SparkGremlinPlugin.java | 50 +++++++++++++++++++-
.../spark/process/HadoopGraphProvider.java | 3 --
6 files changed, 172 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8de580e6/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index ba0e75e..c05d2ad 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -43,7 +43,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -109,24 +108,6 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- /// giraph configuration
- //put(GiraphConstants.MIN_WORKERS, 1);
- //put(GiraphConstants.MAX_WORKERS, 1);
- //put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
- //put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
- //put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- //put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- //put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
- //put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
- //put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
- //put("mapred.reduce.tasks", 4);
- //put("giraph.vertexOutputFormatThreadSafe", false);
- //put("giraph.numOutputThreads", 3);
-
- /// spark configuration
- put("spark.master", "local[4]");
- put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- // put("spark.kryo.registrationRequired",true);
}};
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8de580e6/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index f1c696e..a0bd25e 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -17,7 +17,6 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
@@ -34,6 +33,20 @@
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-core</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
@@ -65,13 +78,21 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>jsp-api-2.1</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<!-- SPARK GRAPH COMPUTER -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>1.2.1</version>
+ <version>1.4.1</version>
<exclusions>
<!-- self conflicts -->
<exclusion>
@@ -83,6 +104,10 @@
<artifactId>scala-library</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
@@ -107,6 +132,10 @@
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.ivy</groupId>
+ <artifactId>ivy</artifactId>
+ </exclusion>
<!-- gremlin-groovy conflicts -->
<exclusion>
<groupId>jline</groupId>
@@ -120,7 +149,7 @@
<!-- lgpl conflicts -->
<exclusion>
<groupId>com.google.code.findbugs</groupId>
- <artifactId>findbugs</artifactId>
+ <artifactId>jsr305</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -130,6 +159,11 @@
<artifactId>scala-library</artifactId>
<version>2.10.3</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.4.4</version>
+ </dependency>
<!-- TEST -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8de580e6/spark-gremlin/src/assembly/hadoop-job.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/assembly/hadoop-job.xml b/spark-gremlin/src/assembly/hadoop-job.xml
new file mode 100644
index 0000000..3093016
--- /dev/null
+++ b/spark-gremlin/src/assembly/hadoop-job.xml
@@ -0,0 +1,39 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<assembly>
+ <id>job</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <unpack>false</unpack>
+ <scope>runtime</scope>
+ <outputDirectory>lib</outputDirectory>
+ <excludes>
+ <exclude>${groupId}:${artifactId}</exclude>
+ </excludes>
+ </dependencySet>
+ <dependencySet>
+ <unpack>true</unpack>
+ <includes>
+ <include>${groupId}:${artifactId}</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8de580e6/spark-gremlin/src/assembly/standalone.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/assembly/standalone.xml b/spark-gremlin/src/assembly/standalone.xml
new file mode 100644
index 0000000..fa7cef1
--- /dev/null
+++ b/spark-gremlin/src/assembly/standalone.xml
@@ -0,0 +1,48 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<assembly>
+ <id>standalone</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>src/main/bin</directory>
+ <outputDirectory>/bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/*.jar</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/lib</outputDirectory>
+ <unpack>false</unpack>
+ <scope>compile</scope>
+ </dependencySet>
+ <dependencySet>
+ <outputDirectory>/lib</outputDirectory>
+ <unpack>false</unpack>
+ <scope>provided</scope>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8de580e6/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
index 0b04300..51a88b3 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
@@ -19,8 +19,54 @@
package org.apache.tinkerpop.gremlin.spark.groovy.plugin;
+import org.apache.tinkerpop.gremlin.groovy.plugin.AbstractGremlinPlugin;
+import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class SparkGremlinPlugin {
-}
+public class SparkGremlinPlugin extends AbstractGremlinPlugin {
+
+ protected static String NAME = "tinkerpop.spark";
+
+ protected static final Set<String> IMPORTS = new HashSet<String>() {{
+ add(IMPORT_SPACE + SparkGraphComputer.class.getPackage().getName() + DOT_STAR);
+ }};
+
+ public SparkGremlinPlugin() {
+ super(true);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public void afterPluginTo(final PluginAcceptor pluginAcceptor) throws PluginInitializationException, IllegalEnvironmentException {
+ pluginAcceptor.addImports(IMPORTS);
+ try {
+ pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", SparkGraphComputer.class.getName()));
+ } catch (final Exception e) {
+ throw new PluginInitializationException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public boolean requireRestart() {
+ return true;
+ }
+
+ @Override
+ public Optional<RemoteAcceptor> remoteAcceptor() {
+ return Optional.empty();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8de580e6/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
index ebda863..cdc4a0b 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
@@ -109,9 +109,6 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
put("mapred.reduce.tasks", 4);
- //put("giraph.vertexOutputFormatThreadSafe", false);
- //put("giraph.numOutputThreads", 3);
-
/// spark configuration
put("spark.master", "local[4]");
put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
[12/18] incubator-tinkerpop git commit: assembly files added and
pom.xmls cleaned up.
Posted by ok...@apache.org.
assembly files added and pom.xmls cleaned up.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/efe085d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/efe085d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/efe085d0
Branch: refs/heads/master
Commit: efe085d0605158e21e428482354989b4da93f885
Parents: 8de580e
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 8 12:50:29 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 8 12:50:29 2015 -0600
----------------------------------------------------------------------
giraph-gremlin/src/assembly/hadoop-job.xml | 39 ++++++++++++++++++++
giraph-gremlin/src/assembly/standalone.xml | 48 +++++++++++++++++++++++++
spark-gremlin/pom.xml | 22 ++++++++++--
3 files changed, 107 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/efe085d0/giraph-gremlin/src/assembly/hadoop-job.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/assembly/hadoop-job.xml b/giraph-gremlin/src/assembly/hadoop-job.xml
new file mode 100644
index 0000000..3093016
--- /dev/null
+++ b/giraph-gremlin/src/assembly/hadoop-job.xml
@@ -0,0 +1,39 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<assembly>
+ <id>job</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <unpack>false</unpack>
+ <scope>runtime</scope>
+ <outputDirectory>lib</outputDirectory>
+ <excludes>
+ <exclude>${groupId}:${artifactId}</exclude>
+ </excludes>
+ </dependencySet>
+ <dependencySet>
+ <unpack>true</unpack>
+ <includes>
+ <include>${groupId}:${artifactId}</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/efe085d0/giraph-gremlin/src/assembly/standalone.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/assembly/standalone.xml b/giraph-gremlin/src/assembly/standalone.xml
new file mode 100644
index 0000000..bad18a0
--- /dev/null
+++ b/giraph-gremlin/src/assembly/standalone.xml
@@ -0,0 +1,48 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<assembly>
+ <id>standalone</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/bin</directory>
+ <outputDirectory>/bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/*.jar</directory>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/lib</outputDirectory>
+ <unpack>false</unpack>
+ <scope>compile</scope>
+ </dependencySet>
+ <dependencySet>
+ <outputDirectory>/lib</outputDirectory>
+ <unpack>false</unpack>
+ <scope>provided</scope>
+ </dependencySet>
+ </dependencySets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/efe085d0/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index a0bd25e..ac6c532 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -36,11 +36,11 @@
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
+ <artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
+ <artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -97,9 +97,17 @@
<!-- self conflicts -->
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</exclusion>
@@ -161,9 +169,19 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.4.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.4.4</version>
</dependency>
+ <!-- <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>2.4.4</version>
+ </dependency>-->
<!-- TEST -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
[13/18] incubator-tinkerpop git commit: More organization of Hadoop
split. Note that Giraph OPT_OUTs are no longer needed. Suspect some jar
conflict was causing the hard thread stop previously. Now that Giraph is
isolated, tests just pass.
Posted by ok...@apache.org.
More organization of Hadoop split. Note that Giraph OPT_OUTs are no longer needed. Suspect some jar conflict was causing the hard thread stop previously. Now that Giraph is isolated, tests just pass.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ffb42a1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ffb42a1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ffb42a1f
Branch: refs/heads/master
Commit: ffb42a1fd410fdbcc664a5e73178bf692449db1f
Parents: efe085d
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 8 15:45:45 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 8 15:45:45 2015 -0600
----------------------------------------------------------------------
.../groovy/plugin/GiraphGremlinPlugin.java | 72 +++++++++
.../groovy/plugin/HadoopGremlinPluginTest.java | 162 +++++++++++++++++++
.../giraph/groovy/plugin/HadoopPluginSuite.java | 36 +++++
.../giraph/process/HadoopGraphProvider.java | 161 ------------------
.../computer/HadoopGiraphGraphProvider.java | 136 +++++++++++++++-
.../GiraphHadoopGremlinPluginIntegrateTest.java | 3 +-
.../gremlin/hadoop/structure/HadoopGraph.java | 20 +--
.../spark/groovy/plugin/SparkGremlinPlugin.java | 2 +-
.../groovy/plugin/HadoopGremlinPluginTest.java | 162 +++++++++++++++++++
.../spark/groovy/plugin/HadoopPluginSuite.java | 36 +++++
.../spark/process/HadoopGraphProvider.java | 151 -----------------
.../computer/HadoopSparkGraphProvider.java | 128 ++++++++++++++-
.../groovy/SparkHadoopGremlinPluginTest.java | 3 +-
13 files changed, 733 insertions(+), 339 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
new file mode 100644
index 0000000..f50b686
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.giraph.groovy.plugin;
+
+import org.apache.tinkerpop.giraph.process.computer.GiraphGraphComputer;
+import org.apache.tinkerpop.gremlin.groovy.plugin.AbstractGremlinPlugin;
+import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphGremlinPlugin extends AbstractGremlinPlugin {
+
+ protected static String NAME = "tinkerpop.spark";
+
+ protected static final Set<String> IMPORTS = new HashSet<String>() {{
+ add(IMPORT_SPACE + GiraphGraphComputer.class.getPackage().getName() + DOT_STAR);
+ }};
+
+ public GiraphGremlinPlugin() {
+ super(true);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public void afterPluginTo(final PluginAcceptor pluginAcceptor) throws PluginInitializationException, IllegalEnvironmentException {
+ pluginAcceptor.addImports(IMPORTS);
+ try {
+ pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", GiraphGraphComputer.class.getName()));
+ } catch (final Exception e) {
+ throw new PluginInitializationException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public boolean requireRestart() {
+ return true;
+ }
+
+ @Override
+ public Optional<RemoteAcceptor> remoteAcceptor() {
+ return Optional.empty();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java
new file mode 100644
index 0000000..d9fd95e
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopGremlinPluginTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.HadoopGiraphGraphProvider;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.util.TestableConsolePluginAcceptor;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPlugin;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopRemoteAcceptor;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.sideEffect.mapreduce.TraverserMapReduce;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+
+public class HadoopGremlinPluginTest extends AbstractGremlinTest {
+
+ @Before
+ public void setupTest() {
+ try {
+ this.console = new TestableConsolePluginAcceptor();
+ final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin();
+ plugin.pluginTo(this.console);
+ this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ ///////////////////
+
+ private HadoopRemoteAcceptor remote;
+ private TestableConsolePluginAcceptor console;
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ //
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()"));
+ assertEquals(6L, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteSugarTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph"));
+ //
+ this.remote.configure(Arrays.asList("useSugar", "true"));
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum"));
+ assertEquals(28.0d, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportHDFSMethods() throws Exception {
+ ////////////////
+ List<String> ls = (List<String>) this.console.eval("hdfs.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ls = (List<String>) this.console.eval("local.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ////////////////
+ this.console.eval("hdfs.copyFromLocal('" + HadoopGiraphGraphProvider.PATHS.get("tinkerpop-classic.txt") + "', 'target/tinkerpop-classic.txt')");
+ assertTrue((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ List<String> head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt')"));
+ assertEquals(6, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 6);
+ }
+ head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt',3)"));
+ assertEquals(3, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 3);
+ }
+ ////////////////
+ this.console.eval("hdfs.rm('target/tinkerpop-classic.txt')");
+ assertFalse((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<Vertex, String> traversal = (Traversal<Vertex, String>) this.remote.submit(Arrays.asList("g.V().hasLabel('person').group('m').by('age').by('name').out('knows').out('created').values('name')"));
+ AbstractGremlinProcessTest.checkResults(Arrays.asList("ripple", "lop"), traversal);
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/m')"));
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "')"));
+ final List<KeyValue<Integer, BulkSet<String>>> mList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/m',ObjectWritable)"));
+ assertEquals(4, mList.size());
+ mList.forEach(keyValue -> {
+ if (keyValue.getKey().equals(29))
+ assertEquals(1l, keyValue.getValue().get("marko"));
+ else if (keyValue.getKey().equals(35))
+ assertEquals(1l, keyValue.getValue().get("peter"));
+ else if (keyValue.getKey().equals(32))
+ assertEquals(1l, keyValue.getValue().get("josh"));
+ else if (keyValue.getKey().equals(27))
+ assertEquals(1l, keyValue.getValue().get("vadas"));
+ else
+ throw new IllegalStateException("The provided key/value is unknown: " + keyValue);
+ });
+ final List<KeyValue<MapReduce.NullObject, Traverser<String>>> traversersList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "',ObjectWritable)"));
+ assertEquals(2, traversersList.size());
+ traversersList.forEach(keyValue -> {
+ assertEquals(MapReduce.NullObject.instance(), keyValue.getKey());
+ final String name = keyValue.getValue().get();
+ assertTrue(name.equals("ripple") || name.equals("lop"));
+ });
+ ////////////////
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java
new file mode 100644
index 0000000..d7d9bc4
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/HadoopPluginSuite.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopPluginSuite extends AbstractGremlinSuite {
+
+ public HadoopPluginSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
+ super(klass, builder, new Class<?>[]{HadoopGremlinPluginTest.class}, new Class<?>[]{HadoopGremlinPluginTest.class}, true, TraversalEngine.Type.COMPUTER);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
deleted file mode 100644
index ae0ae10..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.giraph.process;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class HadoopGraphProvider extends AbstractGraphProvider {
-
- private static final Random RANDOM = new Random();
- private boolean graphSONInput = false;
-
- public static Map<String, String> PATHS = new HashMap<>();
- private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
- add(HadoopEdge.class);
- add(HadoopElement.class);
- add(HadoopGraph.class);
- add(HadoopProperty.class);
- add(HadoopVertex.class);
- add(HadoopVertexProperty.class);
- }};
-
- static {
- try {
- final List<String> kryoResources = Arrays.asList(
- "tinkerpop-modern.kryo",
- "grateful-dead.kryo",
- "tinkerpop-classic.kryo",
- "tinkerpop-crew.kryo");
- for (final String fileName : kryoResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> graphsonResources = Arrays.asList(
- "tinkerpop-modern.json",
- "grateful-dead.json",
- "tinkerpop-classic.json",
- "tinkerpop-crew.json");
- for (final String fileName : graphsonResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> scriptResources = Arrays.asList(
- "tinkerpop-classic.txt",
- "script-input.groovy",
- "script-output.groovy",
- "grateful-dead.txt",
- "script-input-grateful-dead.groovy",
- "script-output-grateful-dead.groovy");
- for (final String fileName : scriptResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- this.graphSONInput = RANDOM.nextBoolean();
- return new HashMap<String, Object>() {{
- put(Graph.GRAPH, HadoopGraph.class.getName());
- put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
- put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- /// giraph configuration
- put(GiraphConstants.MIN_WORKERS, 1);
- put(GiraphConstants.MAX_WORKERS, 1);
- put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
- put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
- put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
- put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
- put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
- put("mapred.reduce.tasks", 4);
- put("giraph.vertexOutputFormatThreadSafe", false);
- put("giraph.numOutputThreads", 3);
- }};
- }
-
- @Override
- public void clear(final Graph graph, final Configuration configuration) throws Exception {
- if (graph != null)
- graph.close();
- }
-
- @Override
- public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
- if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
- }
-
- @Override
- public Set<Class> getImplementations() {
- return IMPLEMENTATION;
- }
-
- public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
- final String type = this.graphSONInput ? "json" : "kryo";
-
- if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
- } else {
- throw new RuntimeException("Could not load graph with " + graphData);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
index 8b124e5..78ea1bb 100644
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
@@ -18,20 +18,152 @@
*/
package org.apache.tinkerpop.gremlin.giraph.process.computer;
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.conf.GiraphConstants;
import org.apache.tinkerpop.giraph.process.computer.GiraphGraphComputer;
+import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.GraphProvider;
-import org.apache.tinkerpop.gremlin.giraph.process.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@GraphProvider.Descriptor(computer = GiraphGraphComputer.class)
-public final class HadoopGiraphGraphProvider extends HadoopGraphProvider {
+public final class HadoopGiraphGraphProvider extends AbstractGraphProvider {
+
+ private static final Random RANDOM = new Random();
+ private boolean graphSONInput = false;
+
+ public static Map<String, String> PATHS = new HashMap<>();
+ private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
+ add(HadoopEdge.class);
+ add(HadoopElement.class);
+ add(HadoopGraph.class);
+ add(HadoopProperty.class);
+ add(HadoopVertex.class);
+ add(HadoopVertexProperty.class);
+ }};
+
+ static {
+ try {
+ final List<String> kryoResources = Arrays.asList(
+ "tinkerpop-modern.kryo",
+ "grateful-dead.kryo",
+ "tinkerpop-classic.kryo",
+ "tinkerpop-crew.kryo");
+ for (final String fileName : kryoResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> graphsonResources = Arrays.asList(
+ "tinkerpop-modern.json",
+ "grateful-dead.json",
+ "tinkerpop-classic.json",
+ "tinkerpop-crew.json");
+ for (final String fileName : graphsonResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> scriptResources = Arrays.asList(
+ "tinkerpop-classic.txt",
+ "script-input.groovy",
+ "script-output.groovy",
+ "grateful-dead.txt",
+ "script-input-grateful-dead.groovy",
+ "script-output-grateful-dead.groovy");
+ for (final String fileName : scriptResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ this.graphSONInput = RANDOM.nextBoolean();
+ return new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ /// giraph configuration
+ put(GiraphConstants.MIN_WORKERS, 1);
+ put(GiraphConstants.MAX_WORKERS, 1);
+ put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
+ put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
+ put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
+ put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
+ put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
+ put("mapred.reduce.tasks", 4);
+ put("giraph.vertexOutputFormatThreadSafe", false);
+ put("giraph.numOutputThreads", 3);
+ }};
+ }
+ @Override
public GraphTraversalSource traversal(final Graph graph) {
return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(GiraphGraphComputer.class)).create(graph);
}
+
+ @Override
+ public void clear(final Graph graph, final Configuration configuration) throws Exception {
+ if (graph != null)
+ graph.close();
+ }
+
+ @Override
+ public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
+ if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
+ }
+
+ @Override
+ public Set<Class> getImplementations() {
+ return IMPLEMENTATION;
+ }
+
+ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
+ final String type = this.graphSONInput ? "json" : "kryo";
+
+ if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
+ } else {
+ throw new RuntimeException("Could not load graph with " + graphData);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
index fd87d2e..28b3692 100644
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
@@ -19,6 +19,7 @@
package org.apache.tinkerpop.gremlin.giraph.process.computer.groovy;
import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.giraph.groovy.plugin.HadoopPluginSuite;
import org.apache.tinkerpop.gremlin.giraph.process.computer.HadoopGiraphGraphProvider;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.junit.runner.RunWith;
@@ -26,7 +27,7 @@ import org.junit.runner.RunWith;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-//@RunWith(HadoopPluginSuite.class)
+@RunWith(HadoopPluginSuite.class)
@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
public class GiraphHadoopGremlinPluginIntegrateTest {
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index ae4f910..6293b31 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -78,7 +78,7 @@ import java.util.stream.Stream;
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
// computers = {GiraphGraphComputer.class})
@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
+ test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@Graph.OptOut(
@@ -137,21 +137,6 @@ import java.util.stream.Stream;
method = "g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldNotAllowNullMemoryKeys",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldNotAllowSettingUndeclaredMemoryKeys",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
- method = "shouldHaveConsistentMemoryVertexPropertiesAndExceptions",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
-@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileTest$Traversals",
method = "g_V_out_out_profile_grateful",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@@ -188,9 +173,6 @@ public final class HadoopGraph implements Graph {
@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) {
- //if (graphComputerClass.equals(GiraphGraphComputer.class))
- // return (C) new GiraphGraphComputer(this);
- //else {
try {
return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
} catch (final Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
index 51a88b3..f5300fe 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
@@ -33,7 +33,7 @@ import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public class SparkGremlinPlugin extends AbstractGremlinPlugin {
+public final class SparkGremlinPlugin extends AbstractGremlinPlugin {
protected static String NAME = "tinkerpop.spark";
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java
new file mode 100644
index 0000000..5931418
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopGremlinPluginTest.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.util.TestableConsolePluginAcceptor;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPlugin;
+import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopRemoteAcceptor;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.traversal.step.sideEffect.mapreduce.TraverserMapReduce;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.BulkSet;
+import org.apache.tinkerpop.gremlin.spark.process.computer.HadoopSparkGraphProvider;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+
+public class HadoopGremlinPluginTest extends AbstractGremlinTest {
+
+ @Before
+ public void setupTest() {
+ try {
+ this.console = new TestableConsolePluginAcceptor();
+ final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin();
+ plugin.pluginTo(this.console);
+ this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ ///////////////////
+
+ private HadoopRemoteAcceptor remote;
+ private TestableConsolePluginAcceptor console;
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ //
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()"));
+ assertEquals(6L, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportRemoteSugarTraversal() throws Exception {
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph"));
+ //
+ this.remote.configure(Arrays.asList("useSugar", "true"));
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum"));
+ assertEquals(28.0d, traversal.next());
+ assertFalse(traversal.hasNext());
+ assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT));
+ }
+
+ @Test
+ @LoadGraphWith(LoadGraphWith.GraphData.MODERN)
+ public void shouldSupportHDFSMethods() throws Exception {
+ ////////////////
+ List<String> ls = (List<String>) this.console.eval("hdfs.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ls = (List<String>) this.console.eval("local.ls()");
+ for (final String line : ls) {
+ assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x"));
+ assertEquals(" ", line.substring(9, 10));
+ }
+ ////////////////
+ this.console.eval("hdfs.copyFromLocal('" + HadoopSparkGraphProvider.PATHS.get("tinkerpop-classic.txt") + "', 'target/tinkerpop-classic.txt')");
+ assertTrue((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ List<String> head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt')"));
+ assertEquals(6, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 6);
+ }
+ head = IteratorUtils.asList(this.console.eval("hdfs.head('target/tinkerpop-classic.txt',3)"));
+ assertEquals(3, head.size());
+ for (final String line : head) {
+ assertEquals(":", line.substring(1, 2));
+ assertTrue(Integer.valueOf(line.substring(0, 1)) <= 3);
+ }
+ ////////////////
+ this.console.eval("hdfs.rm('target/tinkerpop-classic.txt')");
+ assertFalse((Boolean) this.console.eval("hdfs.exists('target/tinkerpop-classic.txt')"));
+ ////////////////
+ this.console.addBinding("graph", this.graph);
+ this.console.addBinding("g", this.g);
+ this.remote.connect(Arrays.asList("graph", "g"));
+ Traversal<Vertex, String> traversal = (Traversal<Vertex, String>) this.remote.submit(Arrays.asList("g.V().hasLabel('person').group('m').by('age').by('name').out('knows').out('created').values('name')"));
+ AbstractGremlinProcessTest.checkResults(Arrays.asList("ripple", "lop"), traversal);
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/m')"));
+ assertTrue((Boolean) this.console.eval("hdfs.exists('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "')"));
+ final List<KeyValue<Integer, BulkSet<String>>> mList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/m',ObjectWritable)"));
+ assertEquals(4, mList.size());
+ mList.forEach(keyValue -> {
+ if (keyValue.getKey().equals(29))
+ assertEquals(1l, keyValue.getValue().get("marko"));
+ else if (keyValue.getKey().equals(35))
+ assertEquals(1l, keyValue.getValue().get("peter"));
+ else if (keyValue.getKey().equals(32))
+ assertEquals(1l, keyValue.getValue().get("josh"));
+ else if (keyValue.getKey().equals(27))
+ assertEquals(1l, keyValue.getValue().get("vadas"));
+ else
+ throw new IllegalStateException("The provided key/value is unknown: " + keyValue);
+ });
+ final List<KeyValue<MapReduce.NullObject, Traverser<String>>> traversersList = IteratorUtils.asList(this.console.eval("hdfs.head('hadoop-gremlin/target/test-output/" + TraverserMapReduce.TRAVERSERS + "',ObjectWritable)"));
+ assertEquals(2, traversersList.size());
+ traversersList.forEach(keyValue -> {
+ assertEquals(MapReduce.NullObject.instance(), keyValue.getKey());
+ final String name = keyValue.getValue().get();
+ assertTrue(name.equals("ripple") || name.equals("lop"));
+ });
+ ////////////////
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java
new file mode 100644
index 0000000..bcce37c
--- /dev/null
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/HadoopPluginSuite.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.RunnerBuilder;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopPluginSuite extends AbstractGremlinSuite {
+
+ public HadoopPluginSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
+ super(klass, builder, new Class<?>[]{HadoopGremlinPluginTest.class}, new Class<?>[]{HadoopGremlinPluginTest.class}, true, TraversalEngine.Type.COMPUTER);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
deleted file mode 100644
index cdc4a0b..0000000
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.spark.process;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
-import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class HadoopGraphProvider extends AbstractGraphProvider {
-
- private static final Random RANDOM = new Random();
- private boolean graphSONInput = false;
-
- public static Map<String, String> PATHS = new HashMap<>();
- private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
- add(HadoopEdge.class);
- add(HadoopElement.class);
- add(HadoopGraph.class);
- add(HadoopProperty.class);
- add(HadoopVertex.class);
- add(HadoopVertexProperty.class);
- }};
-
- static {
- try {
- final List<String> kryoResources = Arrays.asList(
- "tinkerpop-modern.kryo",
- "grateful-dead.kryo",
- "tinkerpop-classic.kryo",
- "tinkerpop-crew.kryo");
- for (final String fileName : kryoResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> graphsonResources = Arrays.asList(
- "tinkerpop-modern.json",
- "grateful-dead.json",
- "tinkerpop-classic.json",
- "tinkerpop-crew.json");
- for (final String fileName : graphsonResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
- }
-
- final List<String> scriptResources = Arrays.asList(
- "tinkerpop-classic.txt",
- "script-input.groovy",
- "script-output.groovy",
- "grateful-dead.txt",
- "script-input-grateful-dead.groovy",
- "script-output-grateful-dead.groovy");
- for (final String fileName : scriptResources) {
- PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- this.graphSONInput = RANDOM.nextBoolean();
- return new HashMap<String, Object>() {{
- put(Graph.GRAPH, HadoopGraph.class.getName());
- put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
- put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
- put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- put("mapred.reduce.tasks", 4);
- /// spark configuration
- put("spark.master", "local[4]");
- put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- // put("spark.kryo.registrationRequired",true);
- }};
- }
-
- @Override
- public void clear(final Graph graph, final Configuration configuration) throws Exception {
- if (graph != null)
- graph.close();
- }
-
- @Override
- public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
- if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
- }
-
- @Override
- public Set<Class> getImplementations() {
- return IMPLEMENTATION;
- }
-
- public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
- final String type = this.graphSONInput ? "json" : "kryo";
-
- if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
- } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
- ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
- } else {
- throw new RuntimeException("Could not load graph with " + graphData);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
index c0c5bba..21b14be 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/HadoopSparkGraphProvider.java
@@ -18,19 +18,141 @@
*/
package org.apache.tinkerpop.gremlin.spark.process.computer;
+import org.apache.commons.configuration.Configuration;
+import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.GraphProvider;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.spark.process.HadoopGraphProvider;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
-public final class HadoopSparkGraphProvider extends HadoopGraphProvider {
+public final class HadoopSparkGraphProvider extends AbstractGraphProvider {
+
+ private static final Random RANDOM = new Random();
+ private boolean graphSONInput = false;
+
+ public static Map<String, String> PATHS = new HashMap<>();
+ private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
+ add(HadoopEdge.class);
+ add(HadoopElement.class);
+ add(HadoopGraph.class);
+ add(HadoopProperty.class);
+ add(HadoopVertex.class);
+ add(HadoopVertexProperty.class);
+ }};
+
+ static {
+ try {
+ final List<String> kryoResources = Arrays.asList(
+ "tinkerpop-modern.kryo",
+ "grateful-dead.kryo",
+ "tinkerpop-classic.kryo",
+ "tinkerpop-crew.kryo");
+ for (final String fileName : kryoResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> graphsonResources = Arrays.asList(
+ "tinkerpop-modern.json",
+ "grateful-dead.json",
+ "tinkerpop-classic.json",
+ "tinkerpop-crew.json");
+ for (final String fileName : graphsonResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> scriptResources = Arrays.asList(
+ "tinkerpop-classic.txt",
+ "script-input.groovy",
+ "script-output.groovy",
+ "grateful-dead.txt",
+ "script-input-grateful-dead.groovy",
+ "script-output-grateful-dead.groovy");
+ for (final String fileName : scriptResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ this.graphSONInput = RANDOM.nextBoolean();
+ return new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ put("mapred.reduce.tasks", 4);
+ /// spark configuration
+ put("spark.master", "local[4]");
+ put("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+ // put("spark.kryo.registrationRequired",true);
+ }};
+ }
+ @Override
public GraphTraversalSource traversal(final Graph graph) {
return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)).create(graph);
}
-}
+
+ @Override
+ public void clear(final Graph graph, final Configuration configuration) throws Exception {
+ if (graph != null)
+ graph.close();
+ }
+
+ @Override
+ public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
+ if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
+ }
+
+ @Override
+ public Set<Class> getImplementations() {
+ return IMPLEMENTATION;
+ }
+
+ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
+ final String type = this.graphSONInput ? "json" : "kryo";
+
+ if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
+ } else {
+ throw new RuntimeException("Could not load graph with " + graphData);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ffb42a1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
index ddfe2d9..f269c4a 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/groovy/SparkHadoopGremlinPluginTest.java
@@ -20,13 +20,14 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.groovy;
import org.apache.tinkerpop.gremlin.GraphProviderClass;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.spark.groovy.plugin.HadoopPluginSuite;
import org.apache.tinkerpop.gremlin.spark.process.computer.HadoopSparkGraphProvider;
import org.junit.runner.RunWith;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-//@RunWith(HadoopPluginSuite.class)
+@RunWith(HadoopPluginSuite.class)
@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
public class SparkHadoopGremlinPluginTest {
}
[05/18] incubator-tinkerpop git commit: Merge branch 'master' into
hadoop_split
Posted by ok...@apache.org.
Merge branch 'master' into hadoop_split
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/718dd630
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/718dd630
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/718dd630
Branch: refs/heads/master
Commit: 718dd630429d34e0467b6388f845e60159f8159a
Parents: 023ba47 bddc44b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Sep 3 13:45:21 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Sep 3 13:45:21 2015 -0600
----------------------------------------------------------------------
.../apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[17/18] incubator-tinkerpop git commit: Got the Giraph/Spark plugins
working. We have an issue where Jackson needs to be shaded. We also can make
it so Giraph/Spark can be used in the same console session -- just need
GIRAPH_JARS and SPARK_JARS to be dif
Posted by ok...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
new file mode 100644
index 0000000..57d5433
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.FileConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.giraph.job.GiraphJob;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.io.GiraphVertexInputFormat;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.io.GiraphVertexOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MapReduceHelper;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.HadoopHelper;
+import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+
+import java.io.File;
+import java.io.NotSerializableException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.stream.Stream;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphGraphComputer extends AbstractHadoopGraphComputer implements GraphComputer, Tool {
+
+ protected GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
+ private MapMemory memory = new MapMemory();
+
+ public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
+ super(hadoopGraph);
+ final Configuration configuration = hadoopGraph.configuration();
+ configuration.getKeys().forEachRemaining(key -> this.giraphConfiguration.set(key, configuration.getProperty(key).toString()));
+ this.giraphConfiguration.setMasterComputeClass(GiraphMemory.class);
+ this.giraphConfiguration.setVertexClass(GiraphComputeVertex.class);
+ this.giraphConfiguration.setWorkerContextClass(GiraphWorkerContext.class);
+ this.giraphConfiguration.setOutEdgesClass(EmptyOutEdges.class);
+ this.giraphConfiguration.setClass(GiraphConstants.VERTEX_ID_CLASS.getKey(), ObjectWritable.class, ObjectWritable.class);
+ this.giraphConfiguration.setClass(GiraphConstants.VERTEX_VALUE_CLASS.getKey(), VertexWritable.class, VertexWritable.class);
+ this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
+ this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
+ this.giraphConfiguration.setVertexOutputFormatClass(GiraphVertexOutputFormat.class);
+ }
+
+ @Override
+ public GraphComputer program(final VertexProgram vertexProgram) {
+ super.program(vertexProgram);
+ this.memory.addVertexProgramMemoryComputeKeys(this.vertexProgram);
+ final BaseConfiguration apacheConfiguration = new BaseConfiguration();
+ vertexProgram.storeState(apacheConfiguration);
+ ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, this.giraphConfiguration);
+ this.vertexProgram.getMessageCombiner().ifPresent(combiner -> this.giraphConfiguration.setCombinerClass(GiraphMessageCombiner.class));
+ return this;
+ }
+
+ @Override
+ public Future<ComputerResult> submit() {
+ final long startTime = System.currentTimeMillis();
+ super.validateStatePriorToExecution();
+ return CompletableFuture.<ComputerResult>supplyAsync(() -> {
+ try {
+ final FileSystem fs = FileSystem.get(this.giraphConfiguration);
+ this.loadJars(fs);
+ fs.delete(new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)), true);
+ ToolRunner.run(this, new String[]{});
+ } catch (final Exception e) {
+ //e.printStackTrace();
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+
+ this.memory.setRuntime(System.currentTimeMillis() - startTime);
+ return new DefaultComputerResult(HadoopHelper.getOutputGraph(this.hadoopGraph, this.resultGraph, this.persist), this.memory.asImmutable());
+ });
+ }
+
+ @Override
+ public int run(final String[] args) {
+ this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(Persist.EDGES));
+ try {
+ // it is possible to run graph computer without a vertex program (and thus, only map reduce jobs if they exist)
+ if (null != this.vertexProgram) {
+ // a way to verify in Giraph whether the traversal will go over the wire or not
+ try {
+ VertexProgram.createVertexProgram(this.hadoopGraph, ConfUtil.makeApacheConfiguration(this.giraphConfiguration));
+ } catch (IllegalStateException e) {
+ if (e.getCause() instanceof NumberFormatException)
+ throw new NotSerializableException("The provided traversal is not serializable and thus, can not be distributed across the cluster");
+ }
+ // prepare the giraph vertex-centric computing job
+ final GiraphJob job = new GiraphJob(this.giraphConfiguration, Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
+ // handle input paths (if any)
+ if (FileInputFormat.class.isAssignableFrom(this.giraphConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
+ final Path inputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
+ if (!FileSystem.get(this.giraphConfiguration).exists(inputPath)) // TODO: what about when the input is not a file input?
+ throw new IllegalArgumentException("The provided input path does not exist: " + inputPath);
+ FileInputFormat.setInputPaths(job.getInternalJob(), inputPath);
+ }
+ // handle output paths
+ final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
+ FileOutputFormat.setOutputPath(job.getInternalJob(), outputPath);
+ job.getInternalJob().setJarByClass(GiraphGraphComputer.class);
+ this.logger.info(Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
+ // execute the job and wait until it completes (if it fails, throw an exception)
+ if (!job.run(true))
+ throw new IllegalStateException("The GiraphGraphComputer job failed -- aborting all subsequent MapReduce jobs"); // how do I get the exception that occured?
+ // add vertex program memory values to the return memory
+ for (final String key : this.vertexProgram.getMemoryComputeKeys()) {
+ final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + key);
+ final ObjectWritableIterator iterator = new ObjectWritableIterator(this.giraphConfiguration, path);
+ if (iterator.hasNext()) {
+ this.memory.set(key, iterator.next().getValue());
+ }
+ FileSystem.get(this.giraphConfiguration).delete(path, true);
+ }
+ final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_ITERATION);
+ this.memory.setIteration((Integer) new ObjectWritableIterator(this.giraphConfiguration, path).next().getValue());
+ FileSystem.get(this.giraphConfiguration).delete(path, true);
+ }
+ // do map reduce jobs
+ this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT_HAS_EDGES, this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, true));
+ for (final MapReduce mapReduce : this.mapReducers) {
+ this.memory.addMapReduceMemoryKey(mapReduce);
+ MapReduceHelper.executeMapReduceJob(mapReduce, this.memory, this.giraphConfiguration);
+ }
+
+ // if no persistence, delete the map reduce output
+ if (this.persist.equals(Persist.NOTHING)) {
+ final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
+ if (FileSystem.get(this.giraphConfiguration).exists(outputPath)) // TODO: what about when the output is not a file output?
+ FileSystem.get(this.giraphConfiguration).delete(outputPath, true);
+ }
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ return 0;
+ }
+
+ @Override
+ public void setConf(final org.apache.hadoop.conf.Configuration configuration) {
+ // TODO: is this necessary to implement?
+ }
+
+ @Override
+ public org.apache.hadoop.conf.Configuration getConf() {
+ return this.giraphConfiguration;
+ }
+
+ private void loadJars(final FileSystem fs) {
+ final String hadoopGremlinLibsRemote = "hadoop-gremlin-libs";
+ if (this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true)) {
+ final String hadoopGremlinLocalLibs = System.getenv(Constants.HADOOP_GREMLIN_LIBS);
+ if (null == hadoopGremlinLocalLibs)
+ this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
+ else {
+ final String[] paths = hadoopGremlinLocalLibs.split(":");
+ for (final String path : paths) {
+ final File file = new File(path);
+ if (file.exists()) {
+ Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> {
+ try {
+ final Path jarFile = new Path(fs.getHomeDirectory() + "/" + hadoopGremlinLibsRemote + "/" + f.getName());
+ fs.copyFromLocalFile(new Path(f.getPath()), jarFile);
+ try {
+ DistributedCache.addArchiveToClassPath(jarFile, this.giraphConfiguration, fs);
+ } catch (final Exception e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ });
+ } else {
+ this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
+ }
+ }
+ }
+ }
+ }
+
+ public static void main(final String[] args) throws Exception {
+ final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
+ new GiraphGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
new file mode 100644
index 0000000..b0cd3f9
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.master.MasterCompute;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphMemory extends MasterCompute implements Memory {
+
+ private VertexProgram<?> vertexProgram;
+ private GiraphWorkerContext worker;
+ private Set<String> memoryKeys;
+ private boolean isMasterCompute = true;
+ private long startTime = System.currentTimeMillis();
+
+ public GiraphMemory() {
+ // Giraph ReflectionUtils requires this to be public at minimum
+ }
+
+ public GiraphMemory(final GiraphWorkerContext worker, final VertexProgram<?> vertexProgram) {
+ this.worker = worker;
+ this.vertexProgram = vertexProgram;
+ this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
+ this.isMasterCompute = false;
+ }
+
+
+ @Override
+ public void initialize() {
+ // do not initialize aggregators here because the getConf() configuration is not available at this point
+ // use compute() initial iteration instead
+ }
+
+ @Override
+ public void compute() {
+ this.isMasterCompute = true;
+ if (0 == this.getSuperstep()) { // setup
+ final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getConf());
+ this.vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
+ this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
+ try {
+ for (final String key : this.memoryKeys) {
+ MemoryHelper.validateKey(key);
+ this.registerPersistentAggregator(key, MemoryAggregator.class);
+ }
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ this.vertexProgram.setup(this);
+ } else {
+ if (this.vertexProgram.terminate(this)) { // terminate
+ // write the memory to HDFS
+ final MapMemory memory = new MapMemory(this);
+ // a hack to get the last iteration memory values to stick
+ this.vertexProgram.terminate(memory);
+ final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
+ if (null != outputLocation) {
+ try {
+ for (final String key : this.keys()) {
+ final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + key), ObjectWritable.class, ObjectWritable.class);
+ writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.get(key)));
+ writer.close();
+ }
+ final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + Constants.HIDDEN_ITERATION), ObjectWritable.class, ObjectWritable.class);
+ writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.getIteration()));
+ writer.close();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ this.haltComputation();
+ }
+ }
+ }
+
+ @Override
+ public int getIteration() {
+ if (this.isMasterCompute) {
+ final int temp = (int) this.getSuperstep();
+ return temp == 0 ? temp : temp - 1;
+ } else {
+ return (int) this.worker.getSuperstep();
+ }
+ }
+
+ @Override
+ public long getRuntime() {
+ return System.currentTimeMillis() - this.startTime;
+ }
+
+ @Override
+ public Set<String> keys() {
+ return this.memoryKeys.stream().filter(this::exists).collect(Collectors.toSet());
+ }
+
+ @Override
+ public boolean exists(final String key) {
+ final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
+ return null != rule.getObject();
+ }
+
+ @Override
+ public <R> R get(final String key) throws IllegalArgumentException {
+ //this.checkKey(key);
+ final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
+ if (null == rule.getObject())
+ throw Memory.Exceptions.memoryDoesNotExist(key);
+ else
+ return rule.getObject();
+ }
+
+ @Override
+ public void set(final String key, Object value) {
+ this.checkKeyValue(key, value);
+ if (this.isMasterCompute)
+ this.setAggregatedValue(key, new Rule(Rule.Operation.SET, value));
+ else
+ this.worker.aggregate(key, new Rule(Rule.Operation.SET, value));
+ }
+
+ @Override
+ public void and(final String key, final boolean bool) {
+ this.checkKeyValue(key, bool);
+ if (this.isMasterCompute) { // only called on setup() and terminate()
+ Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
+ value = null == value ? bool : bool && value;
+ this.setAggregatedValue(key, new Rule(Rule.Operation.AND, value));
+ } else {
+ this.worker.aggregate(key, new Rule(Rule.Operation.AND, bool));
+ }
+ }
+
+ @Override
+ public void or(final String key, final boolean bool) {
+ this.checkKeyValue(key, bool);
+ if (this.isMasterCompute) { // only called on setup() and terminate()
+ Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
+ value = null == value ? bool : bool || value;
+ this.setAggregatedValue(key, new Rule(Rule.Operation.OR, value));
+ } else {
+ this.worker.aggregate(key, new Rule(Rule.Operation.OR, bool));
+ }
+ }
+
+ @Override
+ public void incr(final String key, final long delta) {
+ this.checkKeyValue(key, delta);
+ if (this.isMasterCompute) { // only called on setup() and terminate()
+ Number value = this.<Rule>getAggregatedValue(key).<Number>getObject();
+ value = null == value ? delta : value.longValue() + delta;
+ this.setAggregatedValue(key, new Rule(Rule.Operation.INCR, value));
+ } else {
+ this.worker.aggregate(key, new Rule(Rule.Operation.INCR, delta));
+ }
+ }
+
+ @Override
+ public void write(final DataOutput output) {
+ // no need to serialize the master compute as it gets its data from aggregators
+ // is this true?
+ }
+
+ @Override
+ public void readFields(final DataInput input) {
+ // no need to serialize the master compute as it gets its data from aggregators
+ // is this true?
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.memoryString(this);
+ }
+
+ private void checkKeyValue(final String key, final Object value) {
+ if (!this.memoryKeys.contains(key))
+ throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
+ MemoryHelper.validateValue(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java
new file mode 100644
index 0000000..144d553
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.combiner.Combiner;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphMessageCombiner extends Combiner<ObjectWritable, ObjectWritable> implements ImmutableClassesGiraphConfigurable {
+
+ private MessageCombiner messageCombiner;
+ private ImmutableClassesGiraphConfiguration configuration;
+
+ @Override
+ public void combine(final ObjectWritable vertexIndex, final ObjectWritable originalMessage, final ObjectWritable messageToCombine) {
+ originalMessage.set(originalMessage.isEmpty() ?
+ messageToCombine.get() :
+ this.messageCombiner.combine(originalMessage.get(), messageToCombine.get()));
+ }
+
+ @Override
+ public ObjectWritable createInitialMessage() {
+ return ObjectWritable.empty();
+ }
+
+ @Override
+ public void setConf(final ImmutableClassesGiraphConfiguration configuration) {
+ this.configuration = configuration;
+ final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration);
+ this.messageCombiner = (MessageCombiner) VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().get();
+ }
+
+ @Override
+ public ImmutableClassesGiraphConfiguration getConf() {
+ return this.configuration;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java
new file mode 100644
index 0000000..ed93cf1
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphMessenger<M> implements Messenger<M> {
+
+ private GiraphComputeVertex giraphComputeVertex;
+ private Iterator<ObjectWritable<M>> messages;
+
+ public GiraphMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable<M>> messages) {
+ this.giraphComputeVertex = giraphComputeVertex;
+ this.messages = messages;
+ }
+
+ @Override
+ public Iterator<M> receiveMessages() {
+ return IteratorUtils.map(this.messages, ObjectWritable::get);
+ }
+
+ @Override
+ public void sendMessage(final MessageScope messageScope, final M message) {
+ if (messageScope instanceof MessageScope.Local) {
+ final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
+ final Traversal.Admin<Vertex, Edge> incidentTraversal = GiraphMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.giraphComputeVertex.getValue().get());
+ final Direction direction = GiraphMessenger.getOppositeDirection(incidentTraversal);
+ incidentTraversal.forEachRemaining(edge ->
+ this.giraphComputeVertex.sendMessage(
+ new ObjectWritable<>(edge.vertices(direction).next().id()),
+ new ObjectWritable<>(localMessageScope.getEdgeFunction().apply(message, edge))));
+ } else {
+ final MessageScope.Global globalMessageScope = (MessageScope.Global) messageScope;
+ globalMessageScope.vertices().forEach(vertex ->
+ this.giraphComputeVertex.sendMessage(new ObjectWritable<>(vertex.id()), new ObjectWritable<>(message)));
+ }
+ }
+
+ private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
+ incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
+ return (T) incidentTraversal;
+ }
+
+ private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
+ final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
+ return step.getDirection().opposite();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
new file mode 100644
index 0000000..6469d75
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.giraph.worker.WorkerContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.ImmutableMemory;
+import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
+
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphWorkerContext extends WorkerContext {
+
+ private VertexProgramPool vertexProgramPool;
+ private GiraphMemory memory;
+
+ public GiraphWorkerContext() {
+ // Giraph ReflectionUtils requires this to be public at minimum
+ }
+
+ public void preApplication() throws InstantiationException, IllegalAccessException {
+ final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration());
+ HadoopPools.initialize(apacheConfiguration);
+ final VertexProgram vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
+ this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
+ this.memory = new GiraphMemory(this, vertexProgram);
+ }
+
+ public void postApplication() {
+
+ }
+
+ public void preSuperstep() {
+ this.vertexProgramPool.workerIterationStart(new ImmutableMemory(this.memory));
+ }
+
+ public void postSuperstep() {
+ this.vertexProgramPool.workerIterationEnd(new ImmutableMemory(this.memory));
+ }
+
+ public VertexProgramPool getVertexProgramPool() {
+ return this.vertexProgramPool;
+ }
+
+ public GiraphMemory getMemory() {
+ return this.memory;
+ }
+
+ public GiraphMessenger getMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable> messages) {
+ return new GiraphMessenger(giraphComputeVertex, messages);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java
new file mode 100644
index 0000000..b7da66a
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.giraph.aggregators.Aggregator;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class MemoryAggregator implements Aggregator<Rule> {
+
+ private Object currentObject;
+ private Rule.Operation lastOperation = null;
+
+ public MemoryAggregator() {
+ this.currentObject = null;
+ }
+
+ @Override
+ public Rule getAggregatedValue() {
+ if (null == this.currentObject)
+ return createInitialValue();
+ else if (this.currentObject instanceof Long)
+ return new Rule(Rule.Operation.INCR, this.currentObject);
+ else
+ return new Rule(null == this.lastOperation ? Rule.Operation.NO_OP : this.lastOperation, this.currentObject);
+ }
+
+ @Override
+ public void setAggregatedValue(final Rule rule) {
+ this.currentObject = rule.getObject();
+ }
+
+ @Override
+ public void reset() {
+ this.currentObject = null;
+ }
+
+ @Override
+ public Rule createInitialValue() {
+ return new Rule(Rule.Operation.NO_OP, null);
+ }
+
+ @Override
+ public void aggregate(final Rule ruleWritable) {
+ final Rule.Operation rule = ruleWritable.getOperation();
+ final Object object = ruleWritable.getObject();
+ if (rule != Rule.Operation.NO_OP)
+ this.lastOperation = rule;
+
+ if (null == this.currentObject || rule.equals(Rule.Operation.SET)) {
+ this.currentObject = object;
+ } else {
+ if (rule.equals(Rule.Operation.INCR)) {
+ this.currentObject = (Long) this.currentObject + (Long) object;
+ } else if (rule.equals(Rule.Operation.AND)) {
+ this.currentObject = (Boolean) this.currentObject && (Boolean) object;
+ } else if (rule.equals(Rule.Operation.OR)) {
+ this.currentObject = (Boolean) this.currentObject || (Boolean) object;
+ } else if (rule.equals(Rule.Operation.NO_OP)) {
+ if (object instanceof Boolean) { // only happens when NO_OP booleans are being propagated will this occur
+ if (null == this.lastOperation) {
+ // do nothing ... why?
+ } else if (this.lastOperation.equals(Rule.Operation.AND)) {
+ this.currentObject = (Boolean) this.currentObject && (Boolean) object;
+ } else if (this.lastOperation.equals(Rule.Operation.OR)) {
+ this.currentObject = (Boolean) this.currentObject || (Boolean) object;
+ } else {
+ throw new IllegalStateException("This state should not have occurred: " + ruleWritable);
+ }
+ }
+ } else {
+ throw new IllegalArgumentException("The provided rule is unknown: " + ruleWritable);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexInputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexInputFormat.java
new file mode 100644
index 0000000..01989ff
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexInputFormat.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer.io;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.giraph.io.VertexInputFormat;
+import org.apache.giraph.io.VertexReader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexInputFormat extends VertexInputFormat {
+
+ private InputFormat<NullWritable, VertexWritable> hadoopGraphInputFormat;
+
+ @Override
+ public List<InputSplit> getSplits(final JobContext context, final int minSplitCountHint) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ return this.hadoopGraphInputFormat.getSplits(context);
+ }
+
+ @Override
+ public VertexReader createVertexReader(final InputSplit split, final TaskAttemptContext context) throws IOException {
+ this.constructor(context.getConfiguration());
+ try {
+ return new GiraphVertexReader(this.hadoopGraphInputFormat.createRecordReader(split, context));
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private final void constructor(final Configuration configuration) {
+ if (null == this.hadoopGraphInputFormat) {
+ this.hadoopGraphInputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class, InputFormat.class), configuration);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexOutputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexOutputFormat.java
new file mode 100644
index 0000000..c1360c7
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexOutputFormat.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer.io;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.giraph.io.VertexOutputFormat;
+import org.apache.giraph.io.VertexWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexOutputFormat extends VertexOutputFormat {
+
+ private OutputFormat<NullWritable, VertexWritable> hadoopGraphOutputFormat;
+
+ @Override
+ public VertexWriter createVertexWriter(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ return new GiraphVertexWriter(this.hadoopGraphOutputFormat);
+ }
+
+ @Override
+ public void checkOutputSpecs(final JobContext context) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ this.hadoopGraphOutputFormat.checkOutputSpecs(context);
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ return this.hadoopGraphOutputFormat.getOutputCommitter(context);
+ }
+
+ private final void constructor(final Configuration configuration) {
+ if (null == this.hadoopGraphOutputFormat) {
+ this.hadoopGraphOutputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, OutputFormat.class, OutputFormat.class), configuration);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexReader.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexReader.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexReader.java
new file mode 100644
index 0000000..3313694
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexReader.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer.io;
+
+import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.io.VertexReader;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphComputeVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexReader extends VertexReader {
+
+ private RecordReader<NullWritable, VertexWritable> recordReader;
+
+ public GiraphVertexReader(final RecordReader<NullWritable, VertexWritable> recordReader) {
+ this.recordReader = recordReader;
+ }
+
+ @Override
+ public void initialize(final InputSplit inputSplit, final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.recordReader.initialize(inputSplit, context);
+ }
+
+ @Override
+ public boolean nextVertex() throws IOException, InterruptedException {
+ return this.recordReader.nextKeyValue();
+ }
+
+ @Override
+ public Vertex getCurrentVertex() throws IOException, InterruptedException {
+ return new GiraphComputeVertex(this.recordReader.getCurrentValue());
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.recordReader.close();
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return this.recordReader.getProgress();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexWriter.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexWriter.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexWriter.java
new file mode 100644
index 0000000..5c16d6b
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/io/GiraphVertexWriter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer.io;
+
+import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.io.VertexWriter;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphComputeVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexWriter extends VertexWriter {
+ private final OutputFormat<NullWritable, VertexWritable> outputFormat;
+ private RecordWriter<NullWritable, VertexWritable> recordWriter;
+
+ public GiraphVertexWriter(final OutputFormat<NullWritable, VertexWritable> outputFormat) {
+ this.outputFormat = outputFormat;
+ }
+
+ @Override
+ public void initialize(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.recordWriter = this.outputFormat.getRecordWriter(context);
+ }
+
+ @Override
+ public void close(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.recordWriter.close(context);
+ }
+
+ @Override
+ public void writeVertex(final Vertex vertex) throws IOException, InterruptedException {
+ this.recordWriter.write(NullWritable.get(), ((GiraphComputeVertex) vertex).getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin b/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
new file mode 100644
index 0000000..9c8f3d0
--- /dev/null
+++ b/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
@@ -0,0 +1 @@
+org.apache.tinkerpop.gremlin.giraph.groovy.plugin.GiraphGremlinPlugin
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
index 78ea1bb..09547e8 100644
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
@@ -20,7 +20,6 @@ package org.apache.tinkerpop.gremlin.giraph.process.computer;
import org.apache.commons.configuration.Configuration;
import org.apache.giraph.conf.GiraphConstants;
-import org.apache.tinkerpop.giraph.process.computer.GiraphGraphComputer;
import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 6293b31..80983df 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -66,17 +66,17 @@ import java.util.stream.Stream;
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_knows_b__c_knows_bX",
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
+//computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX",
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
+//computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX",
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- // computers = {GiraphGraphComputer.class})
+// computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX",
@@ -85,17 +85,17 @@ import java.util.stream.Stream;
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_knows_b__c_knows_bX",
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
+//computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX",
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
+//computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX",
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
- //computers = {GiraphGraphComputer.class})
+//computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
@@ -173,22 +173,16 @@ public final class HadoopGraph implements Graph {
@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) {
- try {
- return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
- } catch (final Exception e) {
- throw new IllegalArgumentException(e.getMessage(), e);
- }
- //}
- //else if (graphComputerClass.equals(SparkGraphComputer.class))
- // return (C) new SparkGraphComputer(this);
- //else
- // throw Graph.Exceptions.graphDoesNotSupportProvidedGraphComputer(graphComputerClass);
+ try {
+ return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
+ } catch (final Exception e) {
+ throw Graph.Exceptions.graphDoesNotSupportProvidedGraphComputer(graphComputerClass);
+ }
}
@Override
public GraphComputer compute() {
- return null;
- //return this.compute(GiraphGraphComputer.class);
+ throw new IllegalArgumentException("There is no default GraphComputer for HadoopGraph. Use HadoopGraph.compute(class) to specify the GraphComputer to use.");
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index d49f1d4..eb22f58 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -36,16 +36,8 @@
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -105,10 +97,6 @@
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
@@ -143,6 +131,10 @@
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
<!-- gremlin-core conflicts -->
<exclusion>
<groupId>org.slf4j</groupId>
@@ -185,11 +177,6 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>2.4.4</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.4.4</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin b/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
new file mode 100644
index 0000000..0399720
--- /dev/null
+++ b/spark-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin
@@ -0,0 +1 @@
+org.apache.tinkerpop.gremlin.spark.groovy.plugin.SparkGremlinPlugin
\ No newline at end of file
[04/18] incubator-tinkerpop git commit: Got Hadoop-Gremlin working
again. Pulling out Spark was a disaster as it seems (crazy enough) that Spark
was providing jars Giraph needed. I had to manually add http-commons... it
wasnt in either hadoop-core or Gir
Posted by ok...@apache.org.
Got Hadoop-Gremlin working again. Pulling out Spark was a disaster as it seems (crazy enough) that Spark was providing jars Giraph needed. I had to manually add http-commons... it wasnt in either hadoop-core or Giraph. Weird.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/023ba47e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/023ba47e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/023ba47e
Branch: refs/heads/master
Commit: 023ba47ed0126c103589c189faa95a60547250f0
Parents: 04f5651
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Sep 3 13:37:01 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Sep 3 13:37:01 2015 -0600
----------------------------------------------------------------------
hadoop-gremlin/pom.xml | 43 +++-----------------
.../spark/process/computer/SparkExecutor.java | 18 ++++++++
2 files changed, 23 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/023ba47e/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index 2466da0..674dce0 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -42,25 +42,9 @@ limitations under the License.
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
- <exclusion>
- <groupId>commons-net</groupId>
- <artifactId>commons-net</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-api-2.1</artifactId>
- </exclusion>
</exclusions>
<!--<scope>provided</scope>-->
</dependency>
@@ -89,35 +73,18 @@ limitations under the License.
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
- <!-- spark conflicts -->
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
+ <!-- gremlin-test conflicts -->
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
</exclusions>
</dependency>
<!-- consistent dependencies -->
<dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>2.10.3</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.5.13.Final</version>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <version>3.1</version>
</dependency>
<!-- TEST -->
<dependency>
@@ -246,4 +213,4 @@ limitations under the License.
</plugin>
</plugins>
</build>
-</project>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/023ba47e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
index 288538f..0350101 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.tinkerpop.gremlin.spark.process.computer;
import com.google.common.base.Optional;
[18/18] incubator-tinkerpop git commit: Got the Giraph/Spark plugins
working. We have an issue where Jackson needs to be shaded. We also can make
it so Giraph/Spark can be used in the same console session -- just need
GIRAPH_JARS and SPARK_JARS to be dif
Posted by ok...@apache.org.
Got the Giraph/Spark plugins working. We have an issue where Jackson needs to be shaded. We also can make it so Giraph/Spark can be used in the same console session -- just need GIRAPH_JARS and SPARK_JARS to be different environmental variables. Will merge master/ and work from there --- as well as update asciidocs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/90b28a16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/90b28a16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/90b28a16
Branch: refs/heads/master
Commit: 90b28a16ef6cf0384d4661d17a4e5f756e69666c
Parents: 00963d1
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Sep 9 14:19:58 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Sep 9 14:19:58 2015 -0600
----------------------------------------------------------------------
giraph-gremlin/pom.xml | 42 +---
.../groovy/plugin/GiraphGremlinPlugin.java | 72 ------
.../giraph/process/computer/EmptyOutEdges.java | 80 -------
.../process/computer/GiraphComputeVertex.java | 50 -----
.../process/computer/GiraphGraphComputer.java | 222 -------------------
.../giraph/process/computer/GiraphMemory.java | 218 ------------------
.../process/computer/GiraphMessageCombiner.java | 62 ------
.../process/computer/GiraphMessenger.java | 79 -------
.../process/computer/GiraphWorkerContext.java | 77 -------
.../process/computer/MemoryAggregator.java | 94 --------
.../computer/io/GiraphVertexInputFormat.java | 65 ------
.../computer/io/GiraphVertexOutputFormat.java | 65 ------
.../process/computer/io/GiraphVertexReader.java | 67 ------
.../process/computer/io/GiraphVertexWriter.java | 57 -----
.../groovy/plugin/GiraphGremlinPlugin.java | 72 ++++++
.../giraph/process/computer/EmptyOutEdges.java | 80 +++++++
.../process/computer/GiraphComputeVertex.java | 50 +++++
.../process/computer/GiraphGraphComputer.java | 222 +++++++++++++++++++
.../giraph/process/computer/GiraphMemory.java | 218 ++++++++++++++++++
.../process/computer/GiraphMessageCombiner.java | 62 ++++++
.../process/computer/GiraphMessenger.java | 79 +++++++
.../process/computer/GiraphWorkerContext.java | 77 +++++++
.../process/computer/MemoryAggregator.java | 94 ++++++++
.../computer/io/GiraphVertexInputFormat.java | 65 ++++++
.../computer/io/GiraphVertexOutputFormat.java | 65 ++++++
.../process/computer/io/GiraphVertexReader.java | 67 ++++++
.../process/computer/io/GiraphVertexWriter.java | 57 +++++
...inkerpop.gremlin.groovy.plugin.GremlinPlugin | 1 +
.../computer/HadoopGiraphGraphProvider.java | 1 -
.../gremlin/hadoop/structure/HadoopGraph.java | 30 +--
spark-gremlin/pom.xml | 21 +-
...inkerpop.gremlin.groovy.plugin.GremlinPlugin | 1 +
32 files changed, 1229 insertions(+), 1283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml
index 214e9c4..c5e1a8e 100644
--- a/giraph-gremlin/pom.xml
+++ b/giraph-gremlin/pom.xml
@@ -40,32 +40,6 @@ limitations under the License.
<groupId>org.apache.tinkerpop</groupId>
<artifactId>hadoop-gremlin</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.giraph</groupId>
- <artifactId>giraph-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- GIRAPH GRAPH COMPUTER -->
<dependency>
@@ -92,11 +66,6 @@ limitations under the License.
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
- <!-- gremlin-test conflicts -->
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
</exclusions>
</dependency>
<!-- consistent dependencies -->
@@ -110,13 +79,13 @@ limitations under the License.
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-test</artifactId>
<version>${project.version}</version>
- <scope>test</scope>
<exclusions>
<exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
</exclusion>
</exclusions>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
@@ -207,11 +176,6 @@ limitations under the License.
<manifestEntries>
<Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-core:1.2.1
</Gremlin-Plugin-Dependencies>
- <!-- deletes the servlet-api jar from the path after install - causes conflicts -->
- <Gremlin-Plugin-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
- </Gremlin-Plugin-Paths>
- <Gremlin-Lib-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
- </Gremlin-Lib-Paths>
</manifestEntries>
</archive>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
deleted file mode 100644
index f50b686..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/groovy/plugin/GiraphGremlinPlugin.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.giraph.groovy.plugin;
-
-import org.apache.tinkerpop.giraph.process.computer.GiraphGraphComputer;
-import org.apache.tinkerpop.gremlin.groovy.plugin.AbstractGremlinPlugin;
-import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException;
-import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor;
-import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException;
-import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
-
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.Set;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphGremlinPlugin extends AbstractGremlinPlugin {
-
- protected static String NAME = "tinkerpop.spark";
-
- protected static final Set<String> IMPORTS = new HashSet<String>() {{
- add(IMPORT_SPACE + GiraphGraphComputer.class.getPackage().getName() + DOT_STAR);
- }};
-
- public GiraphGremlinPlugin() {
- super(true);
- }
-
- @Override
- public String getName() {
- return NAME;
- }
-
- @Override
- public void afterPluginTo(final PluginAcceptor pluginAcceptor) throws PluginInitializationException, IllegalEnvironmentException {
- pluginAcceptor.addImports(IMPORTS);
- try {
- pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", GiraphGraphComputer.class.getName()));
- } catch (final Exception e) {
- throw new PluginInitializationException(e.getMessage(), e);
- }
- }
-
- @Override
- public boolean requireRestart() {
- return true;
- }
-
- @Override
- public Optional<RemoteAcceptor> remoteAcceptor() {
- return Optional.empty();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java
deleted file mode 100644
index cb649f6..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.giraph.edge.Edge;
-import org.apache.giraph.edge.OutEdges;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class EmptyOutEdges implements OutEdges<ObjectWritable, NullWritable> {
-
- private static final EmptyOutEdges INSTANCE = new EmptyOutEdges();
-
- public static EmptyOutEdges instance() {
- return INSTANCE;
- }
-
- @Override
- public void initialize(final Iterable<Edge<ObjectWritable, NullWritable>> edges) {
- }
-
- @Override
- public void initialize(final int capacity) {
- }
-
- @Override
- public void initialize() {
- }
-
- @Override
- public void add(final Edge<ObjectWritable, NullWritable> edge) {
- }
-
- @Override
- public void remove(final ObjectWritable targetVertexId) {
- }
-
- @Override
- public int size() {
- return 0;
- }
-
- @Override
- public Iterator<Edge<ObjectWritable, NullWritable>> iterator() {
- return Collections.emptyIterator();
- }
-
- @Override
- public void write(final DataOutput dataOutput) throws IOException {
- }
-
- @Override
- public void readFields(final DataInput dataInput) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java
deleted file mode 100644
index 7f39c46..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphComputeVertex extends Vertex<ObjectWritable, VertexWritable, NullWritable, ObjectWritable> {
-
- public GiraphComputeVertex() {
- }
-
- public GiraphComputeVertex(final VertexWritable vertexWritable) {
- final VertexWritable newWritable = new VertexWritable();
- newWritable.set(vertexWritable.get());
- this.initialize(new ObjectWritable<>(newWritable.get().id()), newWritable, EmptyOutEdges.instance());
-
- }
-
- @Override
- public void compute(final Iterable<ObjectWritable> messages) {
- final GiraphWorkerContext workerContext = (GiraphWorkerContext) this.getWorkerContext();
- final VertexProgram<?> vertexProgram = workerContext.getVertexProgramPool().take();
- vertexProgram.execute(ComputerGraph.vertexProgram(this.getValue().get(), vertexProgram), workerContext.getMessenger(this, messages.iterator()), workerContext.getMemory());
- workerContext.getVertexProgramPool().offer(vertexProgram);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java
deleted file mode 100644
index acd5628..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.FileConfiguration;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.job.GiraphJob;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tinkerpop.giraph.process.computer.io.GiraphVertexInputFormat;
-import org.apache.tinkerpop.giraph.process.computer.io.GiraphVertexOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MapReduceHelper;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.HadoopHelper;
-import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
-
-import java.io.File;
-import java.io.NotSerializableException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
-import java.util.stream.Stream;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphGraphComputer extends AbstractHadoopGraphComputer implements GraphComputer, Tool {
-
- protected GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
- private MapMemory memory = new MapMemory();
-
- public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
- super(hadoopGraph);
- final Configuration configuration = hadoopGraph.configuration();
- configuration.getKeys().forEachRemaining(key -> this.giraphConfiguration.set(key, configuration.getProperty(key).toString()));
- this.giraphConfiguration.setMasterComputeClass(GiraphMemory.class);
- this.giraphConfiguration.setVertexClass(GiraphComputeVertex.class);
- this.giraphConfiguration.setWorkerContextClass(GiraphWorkerContext.class);
- this.giraphConfiguration.setOutEdgesClass(EmptyOutEdges.class);
- this.giraphConfiguration.setClass(GiraphConstants.VERTEX_ID_CLASS.getKey(), ObjectWritable.class, ObjectWritable.class);
- this.giraphConfiguration.setClass(GiraphConstants.VERTEX_VALUE_CLASS.getKey(), VertexWritable.class, VertexWritable.class);
- this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
- this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
- this.giraphConfiguration.setVertexOutputFormatClass(GiraphVertexOutputFormat.class);
- }
-
- @Override
- public GraphComputer program(final VertexProgram vertexProgram) {
- super.program(vertexProgram);
- this.memory.addVertexProgramMemoryComputeKeys(this.vertexProgram);
- final BaseConfiguration apacheConfiguration = new BaseConfiguration();
- vertexProgram.storeState(apacheConfiguration);
- ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, this.giraphConfiguration);
- this.vertexProgram.getMessageCombiner().ifPresent(combiner -> this.giraphConfiguration.setCombinerClass(GiraphMessageCombiner.class));
- return this;
- }
-
- @Override
- public Future<ComputerResult> submit() {
- final long startTime = System.currentTimeMillis();
- super.validateStatePriorToExecution();
- return CompletableFuture.<ComputerResult>supplyAsync(() -> {
- try {
- final FileSystem fs = FileSystem.get(this.giraphConfiguration);
- this.loadJars(fs);
- fs.delete(new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)), true);
- ToolRunner.run(this, new String[]{});
- } catch (final Exception e) {
- //e.printStackTrace();
- throw new IllegalStateException(e.getMessage(), e);
- }
-
- this.memory.setRuntime(System.currentTimeMillis() - startTime);
- return new DefaultComputerResult(HadoopHelper.getOutputGraph(this.hadoopGraph, this.resultGraph, this.persist), this.memory.asImmutable());
- });
- }
-
- @Override
- public int run(final String[] args) {
- this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(Persist.EDGES));
- try {
- // it is possible to run graph computer without a vertex program (and thus, only map reduce jobs if they exist)
- if (null != this.vertexProgram) {
- // a way to verify in Giraph whether the traversal will go over the wire or not
- try {
- VertexProgram.createVertexProgram(this.hadoopGraph, ConfUtil.makeApacheConfiguration(this.giraphConfiguration));
- } catch (IllegalStateException e) {
- if (e.getCause() instanceof NumberFormatException)
- throw new NotSerializableException("The provided traversal is not serializable and thus, can not be distributed across the cluster");
- }
- // prepare the giraph vertex-centric computing job
- final GiraphJob job = new GiraphJob(this.giraphConfiguration, Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
- // handle input paths (if any)
- if (FileInputFormat.class.isAssignableFrom(this.giraphConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
- final Path inputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
- if (!FileSystem.get(this.giraphConfiguration).exists(inputPath)) // TODO: what about when the input is not a file input?
- throw new IllegalArgumentException("The provided input path does not exist: " + inputPath);
- FileInputFormat.setInputPaths(job.getInternalJob(), inputPath);
- }
- // handle output paths
- final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
- FileOutputFormat.setOutputPath(job.getInternalJob(), outputPath);
- job.getInternalJob().setJarByClass(GiraphGraphComputer.class);
- this.logger.info(Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
- // execute the job and wait until it completes (if it fails, throw an exception)
- if (!job.run(true))
- throw new IllegalStateException("The GiraphGraphComputer job failed -- aborting all subsequent MapReduce jobs"); // how do I get the exception that occured?
- // add vertex program memory values to the return memory
- for (final String key : this.vertexProgram.getMemoryComputeKeys()) {
- final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + key);
- final ObjectWritableIterator iterator = new ObjectWritableIterator(this.giraphConfiguration, path);
- if (iterator.hasNext()) {
- this.memory.set(key, iterator.next().getValue());
- }
- FileSystem.get(this.giraphConfiguration).delete(path, true);
- }
- final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_ITERATION);
- this.memory.setIteration((Integer) new ObjectWritableIterator(this.giraphConfiguration, path).next().getValue());
- FileSystem.get(this.giraphConfiguration).delete(path, true);
- }
- // do map reduce jobs
- this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT_HAS_EDGES, this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, true));
- for (final MapReduce mapReduce : this.mapReducers) {
- this.memory.addMapReduceMemoryKey(mapReduce);
- MapReduceHelper.executeMapReduceJob(mapReduce, this.memory, this.giraphConfiguration);
- }
-
- // if no persistence, delete the map reduce output
- if (this.persist.equals(Persist.NOTHING)) {
- final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
- if (FileSystem.get(this.giraphConfiguration).exists(outputPath)) // TODO: what about when the output is not a file output?
- FileSystem.get(this.giraphConfiguration).delete(outputPath, true);
- }
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- return 0;
- }
-
- @Override
- public void setConf(final org.apache.hadoop.conf.Configuration configuration) {
- // TODO: is this necessary to implement?
- }
-
- @Override
- public org.apache.hadoop.conf.Configuration getConf() {
- return this.giraphConfiguration;
- }
-
- private void loadJars(final FileSystem fs) {
- final String hadoopGremlinLibsRemote = "hadoop-gremlin-libs";
- if (this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true)) {
- final String hadoopGremlinLocalLibs = System.getenv(Constants.HADOOP_GREMLIN_LIBS);
- if (null == hadoopGremlinLocalLibs)
- this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
- else {
- final String[] paths = hadoopGremlinLocalLibs.split(":");
- for (final String path : paths) {
- final File file = new File(path);
- if (file.exists()) {
- Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> {
- try {
- final Path jarFile = new Path(fs.getHomeDirectory() + "/" + hadoopGremlinLibsRemote + "/" + f.getName());
- fs.copyFromLocalFile(new Path(f.getPath()), jarFile);
- try {
- DistributedCache.addArchiveToClassPath(jarFile, this.giraphConfiguration, fs);
- } catch (final Exception e) {
- throw new RuntimeException(e.getMessage(), e);
- }
- } catch (Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- });
- } else {
- this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
- }
- }
- }
- }
- }
-
- public static void main(final String[] args) throws Exception {
- final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
- new GiraphGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java
deleted file mode 100644
index 5a56bd3..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.master.MasterCompute;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
-import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMemory extends MasterCompute implements Memory {
-
- private VertexProgram<?> vertexProgram;
- private GiraphWorkerContext worker;
- private Set<String> memoryKeys;
- private boolean isMasterCompute = true;
- private long startTime = System.currentTimeMillis();
-
- public GiraphMemory() {
- // Giraph ReflectionUtils requires this to be public at minimum
- }
-
- public GiraphMemory(final GiraphWorkerContext worker, final VertexProgram<?> vertexProgram) {
- this.worker = worker;
- this.vertexProgram = vertexProgram;
- this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
- this.isMasterCompute = false;
- }
-
-
- @Override
- public void initialize() {
- // do not initialize aggregators here because the getConf() configuration is not available at this point
- // use compute() initial iteration instead
- }
-
- @Override
- public void compute() {
- this.isMasterCompute = true;
- if (0 == this.getSuperstep()) { // setup
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getConf());
- this.vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
- try {
- for (final String key : this.memoryKeys) {
- MemoryHelper.validateKey(key);
- this.registerPersistentAggregator(key, MemoryAggregator.class);
- }
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- this.vertexProgram.setup(this);
- } else {
- if (this.vertexProgram.terminate(this)) { // terminate
- // write the memory to HDFS
- final MapMemory memory = new MapMemory(this);
- // a hack to get the last iteration memory values to stick
- this.vertexProgram.terminate(memory);
- final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
- if (null != outputLocation) {
- try {
- for (final String key : this.keys()) {
- final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + key), ObjectWritable.class, ObjectWritable.class);
- writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.get(key)));
- writer.close();
- }
- final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + Constants.HIDDEN_ITERATION), ObjectWritable.class, ObjectWritable.class);
- writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.getIteration()));
- writer.close();
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
- this.haltComputation();
- }
- }
- }
-
- @Override
- public int getIteration() {
- if (this.isMasterCompute) {
- final int temp = (int) this.getSuperstep();
- return temp == 0 ? temp : temp - 1;
- } else {
- return (int) this.worker.getSuperstep();
- }
- }
-
- @Override
- public long getRuntime() {
- return System.currentTimeMillis() - this.startTime;
- }
-
- @Override
- public Set<String> keys() {
- return this.memoryKeys.stream().filter(this::exists).collect(Collectors.toSet());
- }
-
- @Override
- public boolean exists(final String key) {
- final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
- return null != rule.getObject();
- }
-
- @Override
- public <R> R get(final String key) throws IllegalArgumentException {
- //this.checkKey(key);
- final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
- if (null == rule.getObject())
- throw Memory.Exceptions.memoryDoesNotExist(key);
- else
- return rule.getObject();
- }
-
- @Override
- public void set(final String key, Object value) {
- this.checkKeyValue(key, value);
- if (this.isMasterCompute)
- this.setAggregatedValue(key, new Rule(Rule.Operation.SET, value));
- else
- this.worker.aggregate(key, new Rule(Rule.Operation.SET, value));
- }
-
- @Override
- public void and(final String key, final boolean bool) {
- this.checkKeyValue(key, bool);
- if (this.isMasterCompute) { // only called on setup() and terminate()
- Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
- value = null == value ? bool : bool && value;
- this.setAggregatedValue(key, new Rule(Rule.Operation.AND, value));
- } else {
- this.worker.aggregate(key, new Rule(Rule.Operation.AND, bool));
- }
- }
-
- @Override
- public void or(final String key, final boolean bool) {
- this.checkKeyValue(key, bool);
- if (this.isMasterCompute) { // only called on setup() and terminate()
- Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
- value = null == value ? bool : bool || value;
- this.setAggregatedValue(key, new Rule(Rule.Operation.OR, value));
- } else {
- this.worker.aggregate(key, new Rule(Rule.Operation.OR, bool));
- }
- }
-
- @Override
- public void incr(final String key, final long delta) {
- this.checkKeyValue(key, delta);
- if (this.isMasterCompute) { // only called on setup() and terminate()
- Number value = this.<Rule>getAggregatedValue(key).<Number>getObject();
- value = null == value ? delta : value.longValue() + delta;
- this.setAggregatedValue(key, new Rule(Rule.Operation.INCR, value));
- } else {
- this.worker.aggregate(key, new Rule(Rule.Operation.INCR, delta));
- }
- }
-
- @Override
- public void write(final DataOutput output) {
- // no need to serialize the master compute as it gets its data from aggregators
- // is this true?
- }
-
- @Override
- public void readFields(final DataInput input) {
- // no need to serialize the master compute as it gets its data from aggregators
- // is this true?
- }
-
- @Override
- public String toString() {
- return StringFactory.memoryString(this);
- }
-
- private void checkKeyValue(final String key, final Object value) {
- if (!this.memoryKeys.contains(key))
- throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
- MemoryHelper.validateValue(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java
deleted file mode 100644
index 8da835d..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.combiner.Combiner;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMessageCombiner extends Combiner<ObjectWritable, ObjectWritable> implements ImmutableClassesGiraphConfigurable {
-
- private MessageCombiner messageCombiner;
- private ImmutableClassesGiraphConfiguration configuration;
-
- @Override
- public void combine(final ObjectWritable vertexIndex, final ObjectWritable originalMessage, final ObjectWritable messageToCombine) {
- originalMessage.set(originalMessage.isEmpty() ?
- messageToCombine.get() :
- this.messageCombiner.combine(originalMessage.get(), messageToCombine.get()));
- }
-
- @Override
- public ObjectWritable createInitialMessage() {
- return ObjectWritable.empty();
- }
-
- @Override
- public void setConf(final ImmutableClassesGiraphConfiguration configuration) {
- this.configuration = configuration;
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration);
- this.messageCombiner = (MessageCombiner) VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().get();
- }
-
- @Override
- public ImmutableClassesGiraphConfiguration getConf() {
- return this.configuration;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java
deleted file mode 100644
index ca3e100..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMessenger<M> implements Messenger<M> {
-
- private GiraphComputeVertex giraphComputeVertex;
- private Iterator<ObjectWritable<M>> messages;
-
- public GiraphMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable<M>> messages) {
- this.giraphComputeVertex = giraphComputeVertex;
- this.messages = messages;
- }
-
- @Override
- public Iterator<M> receiveMessages() {
- return IteratorUtils.map(this.messages, ObjectWritable::get);
- }
-
- @Override
- public void sendMessage(final MessageScope messageScope, final M message) {
- if (messageScope instanceof MessageScope.Local) {
- final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
- final Traversal.Admin<Vertex, Edge> incidentTraversal = GiraphMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.giraphComputeVertex.getValue().get());
- final Direction direction = GiraphMessenger.getOppositeDirection(incidentTraversal);
- incidentTraversal.forEachRemaining(edge ->
- this.giraphComputeVertex.sendMessage(
- new ObjectWritable<>(edge.vertices(direction).next().id()),
- new ObjectWritable<>(localMessageScope.getEdgeFunction().apply(message, edge))));
- } else {
- final MessageScope.Global globalMessageScope = (MessageScope.Global) messageScope;
- globalMessageScope.vertices().forEach(vertex ->
- this.giraphComputeVertex.sendMessage(new ObjectWritable<>(vertex.id()), new ObjectWritable<>(message)));
- }
- }
-
- private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
- incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
- return (T) incidentTraversal;
- }
-
- private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
- final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
- return step.getDirection().opposite();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java
deleted file mode 100644
index 6ff7dc0..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.ImmutableMemory;
-import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
-
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphWorkerContext extends WorkerContext {
-
- private VertexProgramPool vertexProgramPool;
- private GiraphMemory memory;
-
- public GiraphWorkerContext() {
- // Giraph ReflectionUtils requires this to be public at minimum
- }
-
- public void preApplication() throws InstantiationException, IllegalAccessException {
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration());
- HadoopPools.initialize(apacheConfiguration);
- final VertexProgram vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
- this.memory = new GiraphMemory(this, vertexProgram);
- }
-
- public void postApplication() {
-
- }
-
- public void preSuperstep() {
- this.vertexProgramPool.workerIterationStart(new ImmutableMemory(this.memory));
- }
-
- public void postSuperstep() {
- this.vertexProgramPool.workerIterationEnd(new ImmutableMemory(this.memory));
- }
-
- public VertexProgramPool getVertexProgramPool() {
- return this.vertexProgramPool;
- }
-
- public GiraphMemory getMemory() {
- return this.memory;
- }
-
- public GiraphMessenger getMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable> messages) {
- return new GiraphMessenger(giraphComputeVertex, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java
deleted file mode 100644
index 6526fab..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer;
-
-import org.apache.giraph.aggregators.Aggregator;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class MemoryAggregator implements Aggregator<Rule> {
-
- private Object currentObject;
- private Rule.Operation lastOperation = null;
-
- public MemoryAggregator() {
- this.currentObject = null;
- }
-
- @Override
- public Rule getAggregatedValue() {
- if (null == this.currentObject)
- return createInitialValue();
- else if (this.currentObject instanceof Long)
- return new Rule(Rule.Operation.INCR, this.currentObject);
- else
- return new Rule(null == this.lastOperation ? Rule.Operation.NO_OP : this.lastOperation, this.currentObject);
- }
-
- @Override
- public void setAggregatedValue(final Rule rule) {
- this.currentObject = rule.getObject();
- }
-
- @Override
- public void reset() {
- this.currentObject = null;
- }
-
- @Override
- public Rule createInitialValue() {
- return new Rule(Rule.Operation.NO_OP, null);
- }
-
- @Override
- public void aggregate(final Rule ruleWritable) {
- final Rule.Operation rule = ruleWritable.getOperation();
- final Object object = ruleWritable.getObject();
- if (rule != Rule.Operation.NO_OP)
- this.lastOperation = rule;
-
- if (null == this.currentObject || rule.equals(Rule.Operation.SET)) {
- this.currentObject = object;
- } else {
- if (rule.equals(Rule.Operation.INCR)) {
- this.currentObject = (Long) this.currentObject + (Long) object;
- } else if (rule.equals(Rule.Operation.AND)) {
- this.currentObject = (Boolean) this.currentObject && (Boolean) object;
- } else if (rule.equals(Rule.Operation.OR)) {
- this.currentObject = (Boolean) this.currentObject || (Boolean) object;
- } else if (rule.equals(Rule.Operation.NO_OP)) {
- if (object instanceof Boolean) { // only happens when NO_OP booleans are being propagated will this occur
- if (null == this.lastOperation) {
- // do nothing ... why?
- } else if (this.lastOperation.equals(Rule.Operation.AND)) {
- this.currentObject = (Boolean) this.currentObject && (Boolean) object;
- } else if (this.lastOperation.equals(Rule.Operation.OR)) {
- this.currentObject = (Boolean) this.currentObject || (Boolean) object;
- } else {
- throw new IllegalStateException("This state should not have occurred: " + ruleWritable);
- }
- }
- } else {
- throw new IllegalArgumentException("The provided rule is unknown: " + ruleWritable);
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java
deleted file mode 100644
index 2b3b723..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer.io;
-
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.giraph.io.VertexInputFormat;
-import org.apache.giraph.io.VertexReader;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexInputFormat extends VertexInputFormat {
-
- private InputFormat<NullWritable, VertexWritable> hadoopGraphInputFormat;
-
- @Override
- public List<InputSplit> getSplits(final JobContext context, final int minSplitCountHint) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- return this.hadoopGraphInputFormat.getSplits(context);
- }
-
- @Override
- public VertexReader createVertexReader(final InputSplit split, final TaskAttemptContext context) throws IOException {
- this.constructor(context.getConfiguration());
- try {
- return new GiraphVertexReader(this.hadoopGraphInputFormat.createRecordReader(split, context));
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-
- private final void constructor(final Configuration configuration) {
- if (null == this.hadoopGraphInputFormat) {
- this.hadoopGraphInputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class, InputFormat.class), configuration);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java
deleted file mode 100644
index 0135bd5..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer.io;
-
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.giraph.io.VertexOutputFormat;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexOutputFormat extends VertexOutputFormat {
-
- private OutputFormat<NullWritable, VertexWritable> hadoopGraphOutputFormat;
-
- @Override
- public VertexWriter createVertexWriter(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- return new GiraphVertexWriter(this.hadoopGraphOutputFormat);
- }
-
- @Override
- public void checkOutputSpecs(final JobContext context) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- this.hadoopGraphOutputFormat.checkOutputSpecs(context);
- }
-
- @Override
- public OutputCommitter getOutputCommitter(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- return this.hadoopGraphOutputFormat.getOutputCommitter(context);
- }
-
- private final void constructor(final Configuration configuration) {
- if (null == this.hadoopGraphOutputFormat) {
- this.hadoopGraphOutputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, OutputFormat.class, OutputFormat.class), configuration);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java
deleted file mode 100644
index 7c8273c..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer.io;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.io.VertexReader;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.tinkerpop.giraph.process.computer.GiraphComputeVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexReader extends VertexReader {
-
- private RecordReader<NullWritable, VertexWritable> recordReader;
-
- public GiraphVertexReader(final RecordReader<NullWritable, VertexWritable> recordReader) {
- this.recordReader = recordReader;
- }
-
- @Override
- public void initialize(final InputSplit inputSplit, final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordReader.initialize(inputSplit, context);
- }
-
- @Override
- public boolean nextVertex() throws IOException, InterruptedException {
- return this.recordReader.nextKeyValue();
- }
-
- @Override
- public Vertex getCurrentVertex() throws IOException, InterruptedException {
- return new GiraphComputeVertex(this.recordReader.getCurrentValue());
- }
-
- @Override
- public void close() throws IOException {
- this.recordReader.close();
- }
-
- @Override
- public float getProgress() throws IOException, InterruptedException {
- return this.recordReader.getProgress();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java
deleted file mode 100644
index 812386d..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.giraph.process.computer.io;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.tinkerpop.giraph.process.computer.GiraphComputeVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexWriter extends VertexWriter {
- private final OutputFormat<NullWritable, VertexWritable> outputFormat;
- private RecordWriter<NullWritable, VertexWritable> recordWriter;
-
- public GiraphVertexWriter(final OutputFormat<NullWritable, VertexWritable> outputFormat) {
- this.outputFormat = outputFormat;
- }
-
- @Override
- public void initialize(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordWriter = this.outputFormat.getRecordWriter(context);
- }
-
- @Override
- public void close(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordWriter.close(context);
- }
-
- @Override
- public void writeVertex(final Vertex vertex) throws IOException, InterruptedException {
- this.recordWriter.write(NullWritable.get(), ((GiraphComputeVertex) vertex).getValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java
new file mode 100644
index 0000000..8518871
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.groovy.plugin;
+
+import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer;
+import org.apache.tinkerpop.gremlin.groovy.plugin.AbstractGremlinPlugin;
+import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor;
+import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException;
+import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
+
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphGremlinPlugin extends AbstractGremlinPlugin {
+
+ protected static String NAME = "tinkerpop.giraph";
+
+ protected static final Set<String> IMPORTS = new HashSet<String>() {{
+ add(IMPORT_SPACE + GiraphGraphComputer.class.getPackage().getName() + DOT_STAR);
+ }};
+
+ public GiraphGremlinPlugin() {
+ super(true);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public void afterPluginTo(final PluginAcceptor pluginAcceptor) throws PluginInitializationException, IllegalEnvironmentException {
+ pluginAcceptor.addImports(IMPORTS);
+ /*try {
+ pluginAcceptor.eval(String.format("LoggerFactory.getLogger(%s).setLevel(Level.INFO)", GiraphGraphComputer.class.getName()));
+ } catch (final Exception e) {
+ throw new PluginInitializationException(e.getMessage(), e);
+ }*/
+ }
+
+ @Override
+ public boolean requireRestart() {
+ return true;
+ }
+
+ @Override
+ public Optional<RemoteAcceptor> remoteAcceptor() {
+ return Optional.empty();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java
new file mode 100644
index 0000000..4df4835
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.giraph.edge.Edge;
+import org.apache.giraph.edge.OutEdges;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class EmptyOutEdges implements OutEdges<ObjectWritable, NullWritable> {
+
+ private static final EmptyOutEdges INSTANCE = new EmptyOutEdges();
+
+ public static EmptyOutEdges instance() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void initialize(final Iterable<Edge<ObjectWritable, NullWritable>> edges) {
+ }
+
+ @Override
+ public void initialize(final int capacity) {
+ }
+
+ @Override
+ public void initialize() {
+ }
+
+ @Override
+ public void add(final Edge<ObjectWritable, NullWritable> edge) {
+ }
+
+ @Override
+ public void remove(final ObjectWritable targetVertexId) {
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+
+ @Override
+ public Iterator<Edge<ObjectWritable, NullWritable>> iterator() {
+ return Collections.emptyIterator();
+ }
+
+ @Override
+ public void write(final DataOutput dataOutput) throws IOException {
+ }
+
+ @Override
+ public void readFields(final DataInput dataInput) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/90b28a16/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputeVertex.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputeVertex.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputeVertex.java
new file mode 100644
index 0000000..b8de43a
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputeVertex.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.giraph.graph.Vertex;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphComputeVertex extends Vertex<ObjectWritable, VertexWritable, NullWritable, ObjectWritable> {
+
+ public GiraphComputeVertex() {
+ }
+
+ public GiraphComputeVertex(final VertexWritable vertexWritable) {
+ final VertexWritable newWritable = new VertexWritable();
+ newWritable.set(vertexWritable.get());
+ this.initialize(new ObjectWritable<>(newWritable.get().id()), newWritable, EmptyOutEdges.instance());
+
+ }
+
+ @Override
+ public void compute(final Iterable<ObjectWritable> messages) {
+ final GiraphWorkerContext workerContext = (GiraphWorkerContext) this.getWorkerContext();
+ final VertexProgram<?> vertexProgram = workerContext.getVertexProgramPool().take();
+ vertexProgram.execute(ComputerGraph.vertexProgram(this.getValue().get(), vertexProgram), workerContext.getMessenger(this, messages.iterator()), workerContext.getMemory());
+ workerContext.getVertexProgramPool().offer(vertexProgram);
+ }
+}
[03/18] incubator-tinkerpop git commit: Spark is now isolated into
spark-gremlin package. Tests are passing. A few hacks here and there just to
get things building. However,
it was pretty easy to split apart -- which is pomising.
Posted by ok...@apache.org.
Spark is now isolated into spark-gremlin package. Tests are passing. A few hacks here and there just to get things building. However, it was pretty easy to split apart -- which is pomising.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/04f5651e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/04f5651e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/04f5651e
Branch: refs/heads/master
Commit: 04f5651e8d8e9108a4281d959b71ff161a52c208
Parents: 0977a25
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Sep 3 12:50:24 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Sep 3 12:50:24 2015 -0600
----------------------------------------------------------------------
hadoop-gremlin/pom.xml | 62 -----
.../groovy/plugin/HadoopGremlinPlugin.java | 3 -
.../process/computer/spark/RuleAccumulator.java | 55 ----
.../process/computer/spark/SparkExecutor.java | 222 -----------------
.../computer/spark/SparkGraphComputer.java | 216 ----------------
.../process/computer/spark/SparkMapEmitter.java | 45 ----
.../process/computer/spark/SparkMemory.java | 181 --------------
.../process/computer/spark/SparkMessenger.java | 83 -------
.../computer/spark/SparkReduceEmitter.java | 45 ----
.../computer/spark/io/InputFormatRDD.java | 46 ----
.../process/computer/spark/io/InputRDD.java | 41 ---
.../computer/spark/io/OutputFormatRDD.java | 48 ----
.../process/computer/spark/io/OutputRDD.java | 31 ---
.../computer/spark/payload/MessagePayload.java | 35 ---
.../process/computer/spark/payload/Payload.java | 27 --
.../spark/payload/ViewIncomingPayload.java | 95 -------
.../spark/payload/ViewOutgoingPayload.java | 46 ----
.../computer/spark/payload/ViewPayload.java | 39 ---
.../gremlin/hadoop/structure/HadoopGraph.java | 16 +-
.../spark/HadoopSparkGraphProvider.java | 37 ---
.../SparkGraphComputerProcessIntegrateTest.java | 32 ---
...GraphComputerGroovyProcessIntegrateTest.java | 33 ---
.../groovy/SparkHadoopGremlinPluginTest.java | 33 ---
.../computer/spark/io/ExampleInputRDD.java | 47 ----
.../computer/spark/io/ExampleOutputRDD.java | 45 ----
.../computer/spark/io/InputOutputRDDTest.java | 59 -----
.../process/computer/spark/io/InputRDDTest.java | 54 ----
.../computer/spark/io/OutputRDDTest.java | 62 -----
pom.xml | 1 +
spark-gremlin/pom.xml | 249 +++++++++++++++++++
.../spark/groovy/plugin/SparkGremlinPlugin.java | 26 ++
.../spark/process/computer/RuleAccumulator.java | 55 ++++
.../spark/process/computer/SparkExecutor.java | 202 +++++++++++++++
.../process/computer/SparkGraphComputer.java | 217 ++++++++++++++++
.../spark/process/computer/SparkMapEmitter.java | 45 ++++
.../spark/process/computer/SparkMemory.java | 181 ++++++++++++++
.../spark/process/computer/SparkMessenger.java | 83 +++++++
.../process/computer/SparkReduceEmitter.java | 45 ++++
.../process/computer/io/InputFormatRDD.java | 47 ++++
.../spark/process/computer/io/InputRDD.java | 41 +++
.../process/computer/io/OutputFormatRDD.java | 49 ++++
.../spark/process/computer/io/OutputRDD.java | 31 +++
.../computer/payload/MessagePayload.java | 35 +++
.../spark/process/computer/payload/Payload.java | 27 ++
.../computer/payload/ViewIncomingPayload.java | 95 +++++++
.../computer/payload/ViewOutgoingPayload.java | 46 ++++
.../process/computer/payload/ViewPayload.java | 39 +++
.../spark/process/HadoopGraphProvider.java | 166 +++++++++++++
.../computer/HadoopSparkGraphProvider.java | 36 +++
.../SparkGraphComputerProcessIntegrateTest.java | 32 +++
...GraphComputerGroovyProcessIntegrateTest.java | 33 +++
.../groovy/SparkHadoopGremlinPluginTest.java | 32 +++
.../process/computer/io/ExampleInputRDD.java | 47 ++++
.../process/computer/io/ExampleOutputRDD.java | 45 ++++
.../process/computer/io/InputOutputRDDTest.java | 59 +++++
.../spark/process/computer/io/InputRDDTest.java | 54 ++++
.../process/computer/io/OutputRDDTest.java | 62 +++++
57 files changed, 2091 insertions(+), 1727 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index cb6a050..2466da0 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -108,68 +108,6 @@ limitations under the License.
</exclusion>
</exclusions>
</dependency>
- <!-- SPARK GRAPH COMPUTER -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
- <version>1.2.1</version>
- <exclusions>
- <!-- self conflicts -->
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <!-- gremlin-core conflicts -->
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </exclusion>
- <!-- gremlin-groovy conflicts -->
- <exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </exclusion>
- <!-- hadoop conflicts -->
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- <!-- giraph conflicts -->
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <!-- lgpl conflicts -->
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>findbugs</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<!-- consistent dependencies -->
<dependency>
<groupId>org.scala-lang</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
index a34153e..18c4b32 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
@@ -30,7 +30,6 @@ import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.mapreduce.MapReduceGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
import org.apache.tinkerpop.gremlin.hadoop.structure.hdfs.HDFSTools;
@@ -71,7 +70,6 @@ public final class HadoopGremlinPlugin extends AbstractGremlinPlugin {
add(IMPORT_SPACE + HDFSTools.class.getPackage().getName() + DOT_STAR);
////
add(IMPORT_SPACE + GiraphGraphComputer.class.getPackage().getName() + DOT_STAR);
- add(IMPORT_SPACE + SparkGraphComputer.class.getPackage().getName() + DOT_STAR);
add(IMPORT_SPACE + MapReduceGraphComputer.class.getPackage().getName() + DOT_STAR);
}};
@@ -92,7 +90,6 @@ public final class HadoopGremlinPlugin extends AbstractGremlinPlugin {
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", Job.class.getName()));
///
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", GiraphGraphComputer.class.getName()));
- pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", SparkGraphComputer.class.getName()));
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", MapReduceGraphComputer.class.getName()));
///
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", HadoopGraph.class.getName()));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/RuleAccumulator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/RuleAccumulator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/RuleAccumulator.java
deleted file mode 100644
index 422b676..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/RuleAccumulator.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.spark.AccumulatorParam;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class RuleAccumulator implements AccumulatorParam<Rule> {
-
- @Override
- public Rule addAccumulator(final Rule a, final Rule b) {
- if (a.getOperation().equals(Rule.Operation.NO_OP))
- return b;
- if (b.getOperation().equals(Rule.Operation.NO_OP))
- return a;
- else
- return new Rule(b.getOperation(), b.getOperation().compute(a.getObject(), b.getObject()));
- }
-
- @Override
- public Rule addInPlace(final Rule a, final Rule b) {
- if (a.getOperation().equals(Rule.Operation.NO_OP))
- return b;
- if (b.getOperation().equals(Rule.Operation.NO_OP))
- return a;
- else
- return new Rule(b.getOperation(), b.getOperation().compute(a.getObject(), b.getObject()));
- }
-
- @Override
- public Rule zero(final Rule rule) {
- return new Rule(Rule.Operation.NO_OP, null);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
deleted file mode 100644
index b7268b8..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import com.google.common.base.Optional;
-import org.apache.commons.configuration.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.MessagePayload;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.Payload;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.ViewIncomingPayload;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.ViewOutgoingPayload;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.ViewPayload;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.KeyValue;
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
-import org.apache.tinkerpop.gremlin.structure.util.Attachable;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import scala.Tuple2;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkExecutor {
-
- private static final String[] EMPTY_ARRAY = new String[0];
-
- private SparkExecutor() {
- }
-
- ////////////////////
- // VERTEX PROGRAM //
- ////////////////////
-
- public static <M> JavaPairRDD<Object, ViewIncomingPayload<M>> executeVertexProgramIteration(
- final JavaPairRDD<Object, VertexWritable> graphRDD,
- final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD,
- final SparkMemory memory,
- final Configuration apacheConfiguration) {
-
- final JavaPairRDD<Object, ViewOutgoingPayload<M>> viewOutgoingRDD = (((null == viewIncomingRDD) ?
- graphRDD.mapValues(vertexWritable -> new Tuple2<>(vertexWritable, Optional.<ViewIncomingPayload<M>>absent())) : // first iteration will not have any views or messages
- graphRDD.leftOuterJoin(viewIncomingRDD)) // every other iteration may have views and messages
- // for each partition of vertices
- .mapPartitionsToPair(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
- final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
- final Set<String> elementComputeKeys = workerVertexProgram.getElementComputeKeys(); // the compute keys as a set
- final String[] elementComputeKeysArray = elementComputeKeys.size() == 0 ? EMPTY_ARRAY : elementComputeKeys.toArray(new String[elementComputeKeys.size()]); // the compute keys as an array
- final SparkMessenger<M> messenger = new SparkMessenger<>();
- workerVertexProgram.workerIterationStart(memory.asImmutable()); // start the worker
- return () -> IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
- final StarGraph.StarVertex vertex = vertexViewIncoming._2()._1().get(); // get the vertex from the vertex writable
- // drop any compute properties that are cached in memory
- if (elementComputeKeysArray.length > 0)
- vertex.dropVertexProperties(elementComputeKeysArray);
- final boolean hasViewAndMessages = vertexViewIncoming._2()._2().isPresent(); // if this is the first iteration, then there are no views or messages
- final List<DetachedVertexProperty<Object>> previousView = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getView() : Collections.emptyList();
- final List<M> incomingMessages = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getIncomingMessages() : Collections.emptyList();
- previousView.forEach(property -> property.attach(Attachable.Method.create(vertex))); // attach the view to the vertex
- ///
- messenger.setVertexAndIncomingMessages(vertex, incomingMessages); // set the messenger with the incoming messages
- workerVertexProgram.execute(ComputerGraph.vertexProgram(vertex, workerVertexProgram), messenger, memory); // execute the vertex program on this vertex for this iteration
- ///
- final List<DetachedVertexProperty<Object>> nextView = elementComputeKeysArray.length == 0 ? // not all vertex programs have compute keys
- Collections.emptyList() :
- IteratorUtils.list(IteratorUtils.map(vertex.properties(elementComputeKeysArray), property -> DetachedFactory.detach(property, true)));
- final List<Tuple2<Object, M>> outgoingMessages = messenger.getOutgoingMessages(); // get the outgoing messages
- if (!partitionIterator.hasNext())
- workerVertexProgram.workerIterationEnd(memory.asImmutable()); // if no more vertices in the partition, end the worker's iteration
- return new Tuple2<>(vertex.id(), new ViewOutgoingPayload<>(nextView, outgoingMessages));
- });
- })).setName("viewOutgoingRDD");
-
- // "message pass" by reducing on the vertex object id of the view and message payloads
- final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().orElse(null);
- final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD = viewOutgoingRDD
- .flatMapToPair(tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
- IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())), // emit the view payload
- IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))))) // emit the outgoing message payloads one by one
- .reduceByKey((a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
- if (a instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
- return a;
- } else if (b instanceof ViewIncomingPayload) {
- ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
- return b;
- } else {
- final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
- c.mergePayload(a, messageCombiner);
- c.mergePayload(b, messageCombiner);
- return c;
- }
- })
- .filter(payload -> !(payload._2() instanceof MessagePayload)) // this happens if there is a message to a vertex that does not exist
- .filter(payload -> !((payload._2() instanceof ViewIncomingPayload) && !((ViewIncomingPayload<M>) payload._2()).hasView())) // this happens if there are many messages to a vertex that does not exist
- .mapValues(payload -> payload instanceof ViewIncomingPayload ?
- (ViewIncomingPayload<M>) payload : // this happens if there is a vertex with incoming messages
- new ViewIncomingPayload<>((ViewPayload) payload)); // this happens if there is a vertex with no incoming messages
-
- newViewIncomingRDD.setName("viewIncomingRDD")
- .foreachPartition(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
- }); // need to complete a task so its BSP and the memory for this iteration is updated
- return newViewIncomingRDD;
- }
-
- /////////////////
- // MAP REDUCE //
- ////////////////
-
- public static <M> JavaPairRDD<Object, VertexWritable> prepareGraphRDDForMapReduce(final JavaPairRDD<Object, VertexWritable> graphRDD, final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD, final String[] elementComputeKeys) {
- return (null == viewIncomingRDD) ? // there was no vertex program
- graphRDD.mapValues(vertexWritable -> {
- vertexWritable.get().dropEdges();
- return vertexWritable;
- }) :
- graphRDD.leftOuterJoin(viewIncomingRDD)
- .mapValues(tuple -> {
- final StarGraph.StarVertex vertex = tuple._1().get();
- vertex.dropEdges();
- vertex.dropVertexProperties(elementComputeKeys);
- final List<DetachedVertexProperty<Object>> view = tuple._2().isPresent() ? tuple._2().get().getView() : Collections.emptyList();
- view.forEach(property -> property.attach(Attachable.Method.create(vertex)));
- return tuple._1();
- });
- }
-
- public static <K, V> JavaPairRDD<K, V> executeMap(final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce, final Configuration apacheConfiguration) {
- JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
- final MapReduce<K, V, ?, ?, ?> workerMapReduce = MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- workerMapReduce.workerStart(MapReduce.Stage.MAP);
- final SparkMapEmitter<K, V> mapEmitter = new SparkMapEmitter<>();
- return () -> IteratorUtils.flatMap(partitionIterator, vertexWritable -> {
- workerMapReduce.map(ComputerGraph.mapReduce(vertexWritable._2().get()), mapEmitter);
- if (!partitionIterator.hasNext())
- workerMapReduce.workerEnd(MapReduce.Stage.MAP);
- return mapEmitter.getEmissions();
- });
- });
- if (mapReduce.getMapKeySort().isPresent())
- mapRDD = mapRDD.sortByKey(mapReduce.getMapKeySort().get());
- return mapRDD;
- }
-
- // TODO: public static executeCombine() is this necessary? YES --- we groupByKey in reduce() where we want to combine first.
-
- public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce(final JavaPairRDD<K, V> mapRDD, final MapReduce<K, V, OK, OV, ?> mapReduce, final Configuration apacheConfiguration) {
- JavaPairRDD<OK, OV> reduceRDD = mapRDD.groupByKey().mapPartitionsToPair(partitionIterator -> {
- HadoopPools.initialize(apacheConfiguration);
- final MapReduce<K, V, OK, OV, ?> workerMapReduce = MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- workerMapReduce.workerStart(MapReduce.Stage.REDUCE);
- final SparkReduceEmitter<OK, OV> reduceEmitter = new SparkReduceEmitter<>();
- return () -> IteratorUtils.flatMap(partitionIterator, keyValue -> {
- workerMapReduce.reduce(keyValue._1(), keyValue._2().iterator(), reduceEmitter);
- if (!partitionIterator.hasNext())
- workerMapReduce.workerEnd(MapReduce.Stage.REDUCE);
- return reduceEmitter.getEmissions();
- });
- });
- if (mapReduce.getReduceKeySort().isPresent())
- reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get());
- return reduceRDD;
- }
-
- ///////////////////
- // Input/Output //
- //////////////////
-
- public static void saveMapReduceRDD(final JavaPairRDD<Object, Object> mapReduceRDD, final MapReduce mapReduce, final Memory.Admin memory, final org.apache.hadoop.conf.Configuration hadoopConfiguration) {
- final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION,null);
- if (null != outputLocation) {
- // map back to a Hadoop stream for output
- mapReduceRDD.mapToPair(keyValue -> new Tuple2<>(new ObjectWritable<>(keyValue._1()), new ObjectWritable<>(keyValue._2()))).saveAsNewAPIHadoopFile(outputLocation + "/" + mapReduce.getMemoryKey(),
- ObjectWritable.class,
- ObjectWritable.class,
- SequenceFileOutputFormat.class, hadoopConfiguration);
- // TODO: mapReduce.addResultToMemory(memory, mapReduceRDD.map(tuple -> new KeyValue<>(tuple._1(), tuple._2())).collect().iterator());
- try {
- mapReduce.addResultToMemory(memory, new ObjectWritableIterator(hadoopConfiguration, new Path(outputLocation + "/" + mapReduce.getMemoryKey())));
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
deleted file mode 100644
index eb1e411..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.commons.configuration.ConfigurationUtils;
-import org.apache.commons.configuration.FileConfiguration;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io.InputFormatRDD;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io.InputRDD;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io.OutputFormatRDD;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io.OutputRDD;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.ViewIncomingPayload;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.HadoopHelper;
-import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
-import java.util.stream.Stream;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
-
- public SparkGraphComputer(final HadoopGraph hadoopGraph) {
- super(hadoopGraph);
- }
-
- @Override
- public Future<ComputerResult> submit() {
- super.validateStatePriorToExecution();
- // apache and hadoop configurations that are used throughout the graph computer computation
- final org.apache.commons.configuration.Configuration apacheConfiguration = new HadoopConfiguration(this.hadoopGraph.configuration());
- apacheConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(Persist.EDGES));
- final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(apacheConfiguration);
- if (FileInputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
- try {
- final String inputLocation = FileSystem.get(hadoopConfiguration).getFileStatus(new Path(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION))).getPath().toString();
- apacheConfiguration.setProperty(Constants.MAPRED_INPUT_DIR, inputLocation);
- hadoopConfiguration.set(Constants.MAPRED_INPUT_DIR, inputLocation);
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
-
- // create the completable future
- return CompletableFuture.<ComputerResult>supplyAsync(() -> {
- final long startTime = System.currentTimeMillis();
- SparkMemory memory = null;
- // delete output location
- final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
- if (null != outputLocation) {
- try {
- FileSystem.get(hadoopConfiguration).delete(new Path(outputLocation), true);
- } catch (final IOException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
- // wire up a spark context
- final SparkConf sparkConfiguration = new SparkConf();
- sparkConfiguration.setAppName(Constants.GREMLIN_HADOOP_SPARK_JOB_PREFIX + (null == this.vertexProgram ? "No VertexProgram" : this.vertexProgram) + "[" + this.mapReducers + "]");
- /*final List<Class> classes = new ArrayList<>();
- classes.addAll(IOClasses.getGryoClasses(GryoMapper.build().create()));
- classes.addAll(IOClasses.getSharedHadoopClasses());
- classes.add(ViewPayload.class);
- classes.add(MessagePayload.class);
- classes.add(ViewIncomingPayload.class);
- classes.add(ViewOutgoingPayload.class);
- sparkConfiguration.registerKryoClasses(classes.toArray(new Class[classes.size()]));*/ // TODO: fix for user submitted jars in Spark 1.3.0
-
- // create the spark configuration from the graph computer configuration
- hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
- // execute the vertex program and map reducers and if there is a failure, auto-close the spark context
- try (final JavaSparkContext sparkContext = new JavaSparkContext(sparkConfiguration)) {
- // add the project jars to the cluster
- this.loadJars(sparkContext, hadoopConfiguration);
- // create a message-passing friendly rdd from the input rdd
- final JavaPairRDD<Object, VertexWritable> graphRDD;
- try {
- graphRDD = hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_RDD, InputFormatRDD.class, InputRDD.class)
- .newInstance()
- .readGraphRDD(apacheConfiguration, sparkContext)
- .setName("graphRDD")
- .cache();
- } catch (final InstantiationException | IllegalAccessException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
-
- ////////////////////////////////
- // process the vertex program //
- ////////////////////////////////
- if (null != this.vertexProgram) {
- // set up the vertex program and wire up configurations
- memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
- this.vertexProgram.setup(memory);
- memory.broadcastMemory(sparkContext);
- final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
- this.vertexProgram.storeState(vertexProgramConfiguration);
- ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
- ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
-
- // execute the vertex program
- while (true) {
- memory.setInTask(true);
- viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(graphRDD, viewIncomingRDD, memory, vertexProgramConfiguration);
- memory.setInTask(false);
- if (this.vertexProgram.terminate(memory))
- break;
- else {
- memory.incrIteration();
- memory.broadcastMemory(sparkContext);
- }
- }
- // write the graph rdd using the output rdd
- if (!this.persist.equals(Persist.NOTHING)) {
- try {
- hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, OutputFormatRDD.class, OutputRDD.class)
- .newInstance()
- .writeGraphRDD(apacheConfiguration, graphRDD);
- } catch (final InstantiationException | IllegalAccessException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
- }
-
- final Memory.Admin finalMemory = null == memory ? new MapMemory() : new MapMemory(memory);
-
- //////////////////////////////
- // process the map reducers //
- //////////////////////////////
- if (!this.mapReducers.isEmpty()) {
- final String[] elementComputeKeys = this.vertexProgram == null ? new String[0] : this.vertexProgram.getElementComputeKeys().toArray(new String[this.vertexProgram.getElementComputeKeys().size()]);
- final JavaPairRDD<Object, VertexWritable> mapReduceGraphRDD = SparkExecutor.prepareGraphRDDForMapReduce(graphRDD, viewIncomingRDD, elementComputeKeys).setName("mapReduceGraphRDD").cache();
- for (final MapReduce mapReduce : this.mapReducers) {
- // execute the map reduce job
- final HadoopConfiguration newApacheConfiguration = new HadoopConfiguration(apacheConfiguration);
- mapReduce.storeState(newApacheConfiguration);
- // map
- final JavaPairRDD mapRDD = SparkExecutor.executeMap((JavaPairRDD) mapReduceGraphRDD, mapReduce, newApacheConfiguration).setName("mapRDD");
- // combine TODO: is this really needed
- // reduce
- final JavaPairRDD reduceRDD = (mapReduce.doStage(MapReduce.Stage.REDUCE)) ? SparkExecutor.executeReduce(mapRDD, mapReduce, newApacheConfiguration).setName("reduceRDD") : null;
- // write the map reduce output back to disk (memory)
- SparkExecutor.saveMapReduceRDD(null == reduceRDD ? mapRDD : reduceRDD, mapReduce, finalMemory, hadoopConfiguration);
- }
- }
- // update runtime and return the newly computed graph
- finalMemory.setRuntime(System.currentTimeMillis() - startTime);
- return new DefaultComputerResult(HadoopHelper.getOutputGraph(this.hadoopGraph, this.resultGraph, this.persist), finalMemory.asImmutable());
- }
- });
- }
-
- /////////////////
-
- private void loadJars(final JavaSparkContext sparkContext, final Configuration hadoopConfiguration) {
- if (hadoopConfiguration.getBoolean(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true)) {
- final String hadoopGremlinLocalLibs = System.getenv(Constants.HADOOP_GREMLIN_LIBS);
- if (null == hadoopGremlinLocalLibs)
- this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
- else {
- final String[] paths = hadoopGremlinLocalLibs.split(":");
- for (final String path : paths) {
- final File file = new File(path);
- if (file.exists())
- Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> sparkContext.addJar(f.getAbsolutePath()));
- else
- this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
- }
- }
- }
- }
-
- public static void main(final String[] args) throws Exception {
- final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
- new SparkGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMapEmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMapEmitter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMapEmitter.java
deleted file mode 100644
index 7141259..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMapEmitter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import scala.Tuple2;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkMapEmitter<K, V> implements MapReduce.MapEmitter<K, V> {
-
- private List<Tuple2<K, V>> emissions = new ArrayList<>();
-
- @Override
- public void emit(final K key, final V value) {
- this.emissions.add(new Tuple2<>(key, value));
- }
-
- public Iterator<Tuple2<K, V>> getEmissions() {
- final Iterator<Tuple2<K,V>> iterator = this.emissions.iterator();
- this.emissions = new ArrayList<>();
- return iterator;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMemory.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMemory.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMemory.java
deleted file mode 100644
index e2de405..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMemory.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.spark.Accumulator;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.broadcast.Broadcast;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkMemory implements Memory.Admin, Serializable {
-
- public final Set<String> memoryKeys = new HashSet<>();
- private final AtomicInteger iteration = new AtomicInteger(0); // do these need to be atomics?
- private final AtomicLong runtime = new AtomicLong(0l);
- private final Map<String, Accumulator<Rule>> memory = new HashMap<>();
- private Broadcast<Map<String, Object>> broadcast;
- private boolean inTask = false;
-
- public SparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) {
- if (null != vertexProgram) {
- for (final String key : vertexProgram.getMemoryComputeKeys()) {
- MemoryHelper.validateKey(key);
- this.memoryKeys.add(key);
- }
- }
- for (final MapReduce mapReduce : mapReducers) {
- this.memoryKeys.add(mapReduce.getMemoryKey());
- }
- for (final String key : this.memoryKeys) {
- this.memory.put(key, sparkContext.accumulator(new Rule(Rule.Operation.NO_OP, null), key, new RuleAccumulator()));
- }
- this.broadcast = sparkContext.broadcast(new HashMap<>());
- }
-
- @Override
- public Set<String> keys() {
- if (this.inTask)
- return this.broadcast.getValue().keySet();
- else {
- final Set<String> trueKeys = new HashSet<>();
- this.memory.forEach((key, value) -> {
- if (value.value().getObject() != null)
- trueKeys.add(key);
- });
- return Collections.unmodifiableSet(trueKeys);
- }
- }
-
- @Override
- public void incrIteration() {
- this.iteration.getAndIncrement();
- }
-
- @Override
- public void setIteration(final int iteration) {
- this.iteration.set(iteration);
- }
-
- @Override
- public int getIteration() {
- return this.iteration.get();
- }
-
- @Override
- public void setRuntime(final long runTime) {
- this.runtime.set(runTime);
- }
-
- @Override
- public long getRuntime() {
- return this.runtime.get();
- }
-
- @Override
- public <R> R get(final String key) throws IllegalArgumentException {
- final R r = this.getValue(key);
- if (null == r)
- throw Memory.Exceptions.memoryDoesNotExist(key);
- else
- return r;
- }
-
- @Override
- public void incr(final String key, final long delta) {
- checkKeyValue(key, delta);
- if (this.inTask)
- this.memory.get(key).add(new Rule(Rule.Operation.INCR, delta));
- else
- this.memory.get(key).setValue(new Rule(Rule.Operation.INCR, this.<Long>getValue(key) + delta));
- }
-
- @Override
- public void and(final String key, final boolean bool) {
- checkKeyValue(key, bool);
- if (this.inTask)
- this.memory.get(key).add(new Rule(Rule.Operation.AND, bool));
- else
- this.memory.get(key).setValue(new Rule(Rule.Operation.AND, this.<Boolean>getValue(key) && bool));
- }
-
- @Override
- public void or(final String key, final boolean bool) {
- checkKeyValue(key, bool);
- if (this.inTask)
- this.memory.get(key).add(new Rule(Rule.Operation.OR, bool));
- else
- this.memory.get(key).setValue(new Rule(Rule.Operation.OR, this.<Boolean>getValue(key) || bool));
- }
-
- @Override
- public void set(final String key, final Object value) {
- checkKeyValue(key, value);
- if (this.inTask)
- this.memory.get(key).add(new Rule(Rule.Operation.SET, value));
- else
- this.memory.get(key).setValue(new Rule(Rule.Operation.SET, value));
- }
-
- @Override
- public String toString() {
- return StringFactory.memoryString(this);
- }
-
- protected void setInTask(final boolean inTask) {
- this.inTask = inTask;
- }
-
- protected void broadcastMemory(final JavaSparkContext sparkContext) {
- this.broadcast.destroy(true); // do we need to block?
- final Map<String, Object> toBroadcast = new HashMap<>();
- this.memory.forEach((key, rule) -> {
- if (null != rule.value().getObject())
- toBroadcast.put(key, rule.value().getObject());
- });
- this.broadcast = sparkContext.broadcast(toBroadcast);
- }
-
- private void checkKeyValue(final String key, final Object value) {
- if (!this.memoryKeys.contains(key))
- throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
- MemoryHelper.validateValue(value);
- }
-
- private <R> R getValue(final String key) {
- return this.inTask ? (R) this.broadcast.value().get(key) : (R) this.memory.get(key).value().getObject();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
deleted file mode 100644
index f52843b..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import scala.Tuple2;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkMessenger<M> implements Messenger<M> {
-
- private Vertex vertex;
- private Iterable<M> incomingMessages;
- private List<Tuple2<Object, M>> outgoingMessages = new ArrayList<>();
-
- public void setVertexAndIncomingMessages(final Vertex vertex, final Iterable<M> incomingMessages) {
- this.vertex = vertex;
- this.incomingMessages = incomingMessages;
- this.outgoingMessages = new ArrayList<>();
- }
-
- public List<Tuple2<Object, M>> getOutgoingMessages() {
- return this.outgoingMessages;
- }
-
- @Override
- public Iterator<M> receiveMessages() {
- return this.incomingMessages.iterator();
- }
-
- @Override
- public void sendMessage(final MessageScope messageScope, final M message) {
- if (messageScope instanceof MessageScope.Local) {
- final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
- final Traversal.Admin<Vertex, Edge> incidentTraversal = SparkMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.vertex);
- final Direction direction = SparkMessenger.getOppositeDirection(incidentTraversal);
- incidentTraversal.forEachRemaining(edge -> this.outgoingMessages.add(new Tuple2<>(edge.vertices(direction).next().id(), message)));
- } else {
- ((MessageScope.Global) messageScope).vertices().forEach(v -> this.outgoingMessages.add(new Tuple2<>(v.id(), message)));
- }
- }
-
- ///////////
-
- private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
- incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
- return (T) incidentTraversal;
- }
-
- private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
- final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
- return step.getDirection().opposite();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkReduceEmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkReduceEmitter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkReduceEmitter.java
deleted file mode 100644
index a5d0175..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkReduceEmitter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import scala.Tuple2;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class SparkReduceEmitter<OK, OV> implements MapReduce.ReduceEmitter<OK, OV> {
-
- private List<Tuple2<OK, OV>> emissions = new ArrayList<>();
-
- @Override
- public void emit(final OK key, final OV value) {
- this.emissions.add(new Tuple2<>(key, value));
- }
-
- public Iterator<Tuple2<OK, OV>> getEmissions() {
- final Iterator<Tuple2<OK, OV>> iterator = this.emissions.iterator();
- this.emissions = new ArrayList<>();
- return iterator;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputFormatRDD.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputFormatRDD.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputFormatRDD.java
deleted file mode 100644
index 082c3b6..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputFormatRDD.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class InputFormatRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- final org.apache.hadoop.conf.Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(configuration);
- return sparkContext.newAPIHadoopRDD(hadoopConfiguration,
- (Class<InputFormat<NullWritable, VertexWritable>>) hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class),
- NullWritable.class,
- VertexWritable.class)
- .mapToPair(tuple -> new Tuple2<>(tuple._2().get().id(), new VertexWritable(tuple._2().get())))
- .reduceByKey((a, b) -> a); // if this is not done, then the graph is partitioned and you can have duplicate vertices
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDD.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDD.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDD.java
deleted file mode 100644
index 75f602b..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDD.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-/**
- * An InputRDD is used to read data from the underlying graph system and yield the respective adjacency list.
- * Note that {@link InputFormatRDD} is a type of InputRDD that simply uses the specified {@link org.apache.hadoop.mapreduce.InputFormat} to generate the respective graphRDD.
- *
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public interface InputRDD {
-
- /**
- * Read the graphRDD from the underlying graph system.
- * @param configuration the configuration for the {@link org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer}.
- * @param sparkContext the Spark context with the requisite methods for generating a {@link JavaPairRDD}.
- * @return an adjacency list representation of the underlying graph system.
- */
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputFormatRDD.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputFormatRDD.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputFormatRDD.java
deleted file mode 100644
index 8d64fd2..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputFormatRDD.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import scala.Tuple2;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class OutputFormatRDD implements OutputRDD {
-
- @Override
- public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object, VertexWritable> graphRDD) {
- final org.apache.hadoop.conf.Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(configuration);
- final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION);
- if (null != outputLocation) {
- // map back to a <nullwritable,vertexwritable> stream for output
- graphRDD.mapToPair(tuple -> new Tuple2<>(NullWritable.get(), tuple._2()))
- .saveAsNewAPIHadoopFile(outputLocation + "/" + Constants.HIDDEN_G,
- NullWritable.class,
- VertexWritable.class,
- (Class<OutputFormat<NullWritable, VertexWritable>>) hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, OutputFormat.class), hadoopConfiguration);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDD.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDD.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDD.java
deleted file mode 100644
index 8f6ef7a..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDD.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public interface OutputRDD {
-
- public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object, VertexWritable> graphRDD);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/MessagePayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/MessagePayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/MessagePayload.java
deleted file mode 100644
index 191c9ca..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/MessagePayload.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class MessagePayload<M> implements Payload {
-
- private final M message;
-
- public MessagePayload(final M message) {
- this.message = message;
- }
-
- public M getMessage() {
- return this.message;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/Payload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/Payload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/Payload.java
deleted file mode 100644
index d901df7..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/Payload.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload;
-
-import java.io.Serializable;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public interface Payload extends Serializable {
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewIncomingPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewIncomingPayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewIncomingPayload.java
deleted file mode 100644
index b236fe9..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewIncomingPayload.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload;
-
-import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ViewIncomingPayload<M> implements Payload {
-
- private List<DetachedVertexProperty<Object>> view = null;
- private final List<M> incomingMessages;
-
-
- public ViewIncomingPayload() {
- this.incomingMessages = null;
- }
-
- public ViewIncomingPayload(final MessageCombiner<M> messageCombiner) {
- this.incomingMessages = null == messageCombiner ? new ArrayList<>() : new ArrayList<>(1);
- }
-
- public ViewIncomingPayload(final ViewPayload viewPayload) {
- this.incomingMessages = null;
- this.view = viewPayload.getView();
- }
-
-
- public List<DetachedVertexProperty<Object>> getView() {
- return null == this.view ? Collections.emptyList() : this.view;
- }
-
-
- public List<M> getIncomingMessages() {
- return null == this.incomingMessages ? Collections.emptyList() : this.incomingMessages;
- }
-
- public boolean hasView() {
- return null != view;
- }
-
- ////////////////////
-
-
- private void mergeMessage(final M message, final MessageCombiner<M> messageCombiner) {
- if (this.incomingMessages.isEmpty() || null == messageCombiner)
- this.incomingMessages.add(message);
- else
- this.incomingMessages.set(0, messageCombiner.combine(this.incomingMessages.get(0), message));
- }
-
- private void mergeViewIncomingPayload(final ViewIncomingPayload<M> viewIncomingPayload, final MessageCombiner<M> messageCombiner) {
- if (this.view == null)
- this.view = viewIncomingPayload.view;
- else
- this.view.addAll(viewIncomingPayload.getView());
-
- for (final M message : viewIncomingPayload.getIncomingMessages()) {
- this.mergeMessage(message, messageCombiner);
- }
- }
-
- public void mergePayload(final Payload payload, final MessageCombiner<M> messageCombiner) {
- if (payload instanceof ViewPayload)
- this.view = ((ViewPayload) payload).getView();
- else if (payload instanceof MessagePayload)
- this.mergeMessage(((MessagePayload<M>) payload).getMessage(), messageCombiner);
- else if (payload instanceof ViewIncomingPayload)
- this.mergeViewIncomingPayload((ViewIncomingPayload<M>) payload, messageCombiner);
- else
- throw new IllegalArgumentException("The provided payload is an unsupported merge payload: " + payload);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewOutgoingPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewOutgoingPayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewOutgoingPayload.java
deleted file mode 100644
index c4daf47..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewOutgoingPayload.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload;
-
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
-import scala.Tuple2;
-
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ViewOutgoingPayload<M> implements Payload {
-
- private final List<DetachedVertexProperty<Object>> view;
- private final List<Tuple2<Object,M>> outgoingMessages;
-
- public ViewOutgoingPayload(final List<DetachedVertexProperty<Object>> view, final List<Tuple2<Object,M>> outgoingMessages) {
- this.view = view;
- this.outgoingMessages = outgoingMessages;
- }
-
- public ViewPayload getView() {
- return new ViewPayload(this.view);
- }
-
- public List<Tuple2<Object,M>> getOutgoingMessages() {
- return this.outgoingMessages;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewPayload.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewPayload.java
deleted file mode 100644
index 0ec5ef5..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/payload/ViewPayload.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload;
-
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
-
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ViewPayload implements Payload {
-
- private final List<DetachedVertexProperty<Object>> view;
-
- public ViewPayload(final List<DetachedVertexProperty<Object>> view) {
- this.view = view;
- }
-
- public List<DetachedVertexProperty<Object>> getView() {
- return this.view;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 18d515e..3c3c9b7 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.hadoop.structure.hdfs.HadoopEdgeIterator;
import org.apache.tinkerpop.gremlin.hadoop.structure.hdfs.HadoopVertexIterator;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
@@ -192,10 +191,17 @@ public final class HadoopGraph implements Graph {
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) {
if (graphComputerClass.equals(GiraphGraphComputer.class))
return (C) new GiraphGraphComputer(this);
- else if (graphComputerClass.equals(SparkGraphComputer.class))
- return (C) new SparkGraphComputer(this);
- else
- throw Graph.Exceptions.graphDoesNotSupportProvidedGraphComputer(graphComputerClass);
+ else {
+ try {
+ return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
+ } catch (final Exception e) {
+ throw new IllegalArgumentException(e.getMessage(), e);
+ }
+ }
+ //else if (graphComputerClass.equals(SparkGraphComputer.class))
+ // return (C) new SparkGraphComputer(this);
+ //else
+ // throw Graph.Exceptions.graphDoesNotSupportProvidedGraphComputer(graphComputerClass);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/HadoopSparkGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/HadoopSparkGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/HadoopSparkGraphProvider.java
deleted file mode 100644
index af0d745..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/HadoopSparkGraphProvider.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.tinkerpop.gremlin.GraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@GraphProvider.Descriptor(computer = SparkGraphComputer.class)
-public final class HadoopSparkGraphProvider extends HadoopGraphProvider {
-
- public GraphTraversalSource traversal(final Graph graph) {
- return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)).create(graph);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputerProcessIntegrateTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputerProcessIntegrateTest.java
deleted file mode 100644
index ae729fd..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputerProcessIntegrateTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(ProcessComputerSuite.class)
-@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
-public class SparkGraphComputerProcessIntegrateTest {
-}
[08/18] incubator-tinkerpop git commit: pulled out giraph-gremlin
from hadoop-gremlin. HadoopGremlin builds nicely. Giraph too. Spark is still
causing problems.
Posted by ok...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
new file mode 100644
index 0000000..fd87d2e
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer.groovy;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.HadoopGiraphGraphProvider;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+//@RunWith(HadoopPluginSuite.class)
+@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
+public class GiraphHadoopGremlinPluginIntegrateTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml
index 674dce0..9161046 100644
--- a/hadoop-gremlin/pom.xml
+++ b/hadoop-gremlin/pom.xml
@@ -48,56 +48,12 @@ limitations under the License.
</exclusions>
<!--<scope>provided</scope>-->
</dependency>
- <!-- GIRAPH GRAPH COMPUTER -->
- <dependency>
- <groupId>org.apache.giraph</groupId>
- <artifactId>giraph-core</artifactId>
- <version>1.0.0</version>
- <exclusions>
- <!-- self conflicts -->
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <!-- conflicts with gremlin-core -->
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <!-- gremlin-groovy conflicts -->
- <exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </exclusion>
- <!-- gremlin-test conflicts -->
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- consistent dependencies -->
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.1</version>
- </dependency>
<!-- TEST -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>gremlin-test</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
@@ -112,12 +68,6 @@ limitations under the License.
<scope>test</scope>
</dependency>
</dependencies>
- <repositories>
- <repository>
- <id>hyracks-releases</id>
- <url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public-releases/</url>
- </repository>
- </repositories>
<build>
<directory>${basedir}/target</directory>
<finalName>${project.artifactId}-${project.version}</finalName>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
index 18c4b32..ec03569 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java
@@ -28,7 +28,6 @@ import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor;
import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException;
import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.mapreduce.MapReduceGraphComputer;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
@@ -69,7 +68,6 @@ public final class HadoopGremlinPlugin extends AbstractGremlinPlugin {
add(IMPORT_SPACE + ScriptInputFormat.class.getPackage().getName() + DOT_STAR);
add(IMPORT_SPACE + HDFSTools.class.getPackage().getName() + DOT_STAR);
////
- add(IMPORT_SPACE + GiraphGraphComputer.class.getPackage().getName() + DOT_STAR);
add(IMPORT_SPACE + MapReduceGraphComputer.class.getPackage().getName() + DOT_STAR);
}};
@@ -89,7 +87,6 @@ public final class HadoopGremlinPlugin extends AbstractGremlinPlugin {
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", JobClient.class.getName()));
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", Job.class.getName()));
///
- pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", GiraphGraphComputer.class.getName()));
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", MapReduceGraphComputer.class.getName()));
///
pluginAcceptor.eval(String.format("Logger.getLogger(%s).setLevel(Level.INFO)", HadoopGraph.class.getName()));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java
deleted file mode 100644
index 607f085..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/EmptyOutEdges.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.giraph.edge.Edge;
-import org.apache.giraph.edge.OutEdges;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class EmptyOutEdges implements OutEdges<ObjectWritable, NullWritable> {
-
- private static final EmptyOutEdges INSTANCE = new EmptyOutEdges();
-
- public static EmptyOutEdges instance() {
- return INSTANCE;
- }
-
- @Override
- public void initialize(final Iterable<Edge<ObjectWritable, NullWritable>> edges) {
- }
-
- @Override
- public void initialize(final int capacity) {
- }
-
- @Override
- public void initialize() {
- }
-
- @Override
- public void add(final Edge<ObjectWritable, NullWritable> edge) {
- }
-
- @Override
- public void remove(final ObjectWritable targetVertexId) {
- }
-
- @Override
- public int size() {
- return 0;
- }
-
- @Override
- public Iterator<Edge<ObjectWritable, NullWritable>> iterator() {
- return Collections.emptyIterator();
- }
-
- @Override
- public void write(final DataOutput dataOutput) throws IOException {
- }
-
- @Override
- public void readFields(final DataInput dataInput) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
deleted file mode 100644
index 2581388..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphComputeVertex.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphComputeVertex extends Vertex<ObjectWritable, VertexWritable, NullWritable, ObjectWritable> {
-
- public GiraphComputeVertex() {
- }
-
- public GiraphComputeVertex(final VertexWritable vertexWritable) {
- final VertexWritable newWritable = new VertexWritable();
- newWritable.set(vertexWritable.get());
- this.initialize(new ObjectWritable<>(newWritable.get().id()), newWritable, EmptyOutEdges.instance());
-
- }
-
- @Override
- public void compute(final Iterable<ObjectWritable> messages) {
- final GiraphWorkerContext workerContext = (GiraphWorkerContext) this.getWorkerContext();
- final VertexProgram<?> vertexProgram = workerContext.getVertexProgramPool().take();
- vertexProgram.execute(ComputerGraph.vertexProgram(this.getValue().get(), vertexProgram), workerContext.getMessenger(this, messages.iterator()), workerContext.getMemory());
- workerContext.getVertexProgramPool().offer(vertexProgram);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
deleted file mode 100644
index 6995331..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputer.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.FileConfiguration;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.job.GiraphJob;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io.GiraphVertexInputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io.GiraphVertexOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MapReduceHelper;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.HadoopHelper;
-import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
-
-import java.io.File;
-import java.io.NotSerializableException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Future;
-import java.util.stream.Stream;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphGraphComputer extends AbstractHadoopGraphComputer implements GraphComputer, Tool {
-
- protected GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
- private MapMemory memory = new MapMemory();
-
- public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
- super(hadoopGraph);
- final Configuration configuration = hadoopGraph.configuration();
- configuration.getKeys().forEachRemaining(key -> this.giraphConfiguration.set(key, configuration.getProperty(key).toString()));
- this.giraphConfiguration.setMasterComputeClass(GiraphMemory.class);
- this.giraphConfiguration.setVertexClass(GiraphComputeVertex.class);
- this.giraphConfiguration.setWorkerContextClass(GiraphWorkerContext.class);
- this.giraphConfiguration.setOutEdgesClass(EmptyOutEdges.class);
- this.giraphConfiguration.setClass(GiraphConstants.VERTEX_ID_CLASS.getKey(), ObjectWritable.class, ObjectWritable.class);
- this.giraphConfiguration.setClass(GiraphConstants.VERTEX_VALUE_CLASS.getKey(), VertexWritable.class, VertexWritable.class);
- this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
- this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
- this.giraphConfiguration.setVertexOutputFormatClass(GiraphVertexOutputFormat.class);
- }
-
- @Override
- public GraphComputer program(final VertexProgram vertexProgram) {
- super.program(vertexProgram);
- this.memory.addVertexProgramMemoryComputeKeys(this.vertexProgram);
- final BaseConfiguration apacheConfiguration = new BaseConfiguration();
- vertexProgram.storeState(apacheConfiguration);
- ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, this.giraphConfiguration);
- this.vertexProgram.getMessageCombiner().ifPresent(combiner -> this.giraphConfiguration.setCombinerClass(GiraphMessageCombiner.class));
- return this;
- }
-
- @Override
- public Future<ComputerResult> submit() {
- final long startTime = System.currentTimeMillis();
- super.validateStatePriorToExecution();
- return CompletableFuture.<ComputerResult>supplyAsync(() -> {
- try {
- final FileSystem fs = FileSystem.get(this.giraphConfiguration);
- this.loadJars(fs);
- fs.delete(new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)), true);
- ToolRunner.run(this, new String[]{});
- } catch (final Exception e) {
- //e.printStackTrace();
- throw new IllegalStateException(e.getMessage(), e);
- }
-
- this.memory.setRuntime(System.currentTimeMillis() - startTime);
- return new DefaultComputerResult(HadoopHelper.getOutputGraph(this.hadoopGraph, this.resultGraph, this.persist), this.memory.asImmutable());
- });
- }
-
- @Override
- public int run(final String[] args) {
- this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(Persist.EDGES));
- try {
- // it is possible to run graph computer without a vertex program (and thus, only map reduce jobs if they exist)
- if (null != this.vertexProgram) {
- // a way to verify in Giraph whether the traversal will go over the wire or not
- try {
- VertexProgram.createVertexProgram(this.hadoopGraph, ConfUtil.makeApacheConfiguration(this.giraphConfiguration));
- } catch (IllegalStateException e) {
- if (e.getCause() instanceof NumberFormatException)
- throw new NotSerializableException("The provided traversal is not serializable and thus, can not be distributed across the cluster");
- }
- // prepare the giraph vertex-centric computing job
- final GiraphJob job = new GiraphJob(this.giraphConfiguration, Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
- // handle input paths (if any)
- if (FileInputFormat.class.isAssignableFrom(this.giraphConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
- final Path inputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
- if (!FileSystem.get(this.giraphConfiguration).exists(inputPath)) // TODO: what about when the input is not a file input?
- throw new IllegalArgumentException("The provided input path does not exist: " + inputPath);
- FileInputFormat.setInputPaths(job.getInternalJob(), inputPath);
- }
- // handle output paths
- final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
- FileOutputFormat.setOutputPath(job.getInternalJob(), outputPath);
- job.getInternalJob().setJarByClass(GiraphGraphComputer.class);
- this.logger.info(Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
- // execute the job and wait until it completes (if it fails, throw an exception)
- if (!job.run(true))
- throw new IllegalStateException("The GiraphGraphComputer job failed -- aborting all subsequent MapReduce jobs"); // how do I get the exception that occured?
- // add vertex program memory values to the return memory
- for (final String key : this.vertexProgram.getMemoryComputeKeys()) {
- final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + key);
- final ObjectWritableIterator iterator = new ObjectWritableIterator(this.giraphConfiguration, path);
- if (iterator.hasNext()) {
- this.memory.set(key, iterator.next().getValue());
- }
- FileSystem.get(this.giraphConfiguration).delete(path, true);
- }
- final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_ITERATION);
- this.memory.setIteration((Integer) new ObjectWritableIterator(this.giraphConfiguration, path).next().getValue());
- FileSystem.get(this.giraphConfiguration).delete(path, true);
- }
- // do map reduce jobs
- this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT_HAS_EDGES, this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, true));
- for (final MapReduce mapReduce : this.mapReducers) {
- this.memory.addMapReduceMemoryKey(mapReduce);
- MapReduceHelper.executeMapReduceJob(mapReduce, this.memory, this.giraphConfiguration);
- }
-
- // if no persistence, delete the map reduce output
- if (this.persist.equals(Persist.NOTHING)) {
- final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
- if (FileSystem.get(this.giraphConfiguration).exists(outputPath)) // TODO: what about when the output is not a file output?
- FileSystem.get(this.giraphConfiguration).delete(outputPath, true);
- }
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- return 0;
- }
-
- @Override
- public void setConf(final org.apache.hadoop.conf.Configuration configuration) {
- // TODO: is this necessary to implement?
- }
-
- @Override
- public org.apache.hadoop.conf.Configuration getConf() {
- return this.giraphConfiguration;
- }
-
- private void loadJars(final FileSystem fs) {
- final String hadoopGremlinLibsRemote = "hadoop-gremlin-libs";
- if (this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true)) {
- final String hadoopGremlinLocalLibs = System.getenv(Constants.HADOOP_GREMLIN_LIBS);
- if (null == hadoopGremlinLocalLibs)
- this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
- else {
- final String[] paths = hadoopGremlinLocalLibs.split(":");
- for (final String path : paths) {
- final File file = new File(path);
- if (file.exists()) {
- Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> {
- try {
- final Path jarFile = new Path(fs.getHomeDirectory() + "/" + hadoopGremlinLibsRemote + "/" + f.getName());
- fs.copyFromLocalFile(new Path(f.getPath()), jarFile);
- try {
- DistributedCache.addArchiveToClassPath(jarFile, this.giraphConfiguration, fs);
- } catch (final Exception e) {
- throw new RuntimeException(e.getMessage(), e);
- }
- } catch (Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- });
- } else {
- this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
- }
- }
- }
- }
- }
-
- public static void main(final String[] args) throws Exception {
- final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
- new GiraphGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMemory.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMemory.java
deleted file mode 100644
index c557e42..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMemory.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.master.MasterCompute;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
-import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMemory extends MasterCompute implements Memory {
-
- private VertexProgram<?> vertexProgram;
- private GiraphWorkerContext worker;
- private Set<String> memoryKeys;
- private boolean isMasterCompute = true;
- private long startTime = System.currentTimeMillis();
-
- public GiraphMemory() {
- // Giraph ReflectionUtils requires this to be public at minimum
- }
-
- public GiraphMemory(final GiraphWorkerContext worker, final VertexProgram<?> vertexProgram) {
- this.worker = worker;
- this.vertexProgram = vertexProgram;
- this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
- this.isMasterCompute = false;
- }
-
-
- @Override
- public void initialize() {
- // do not initialize aggregators here because the getConf() configuration is not available at this point
- // use compute() initial iteration instead
- }
-
- @Override
- public void compute() {
- this.isMasterCompute = true;
- if (0 == this.getSuperstep()) { // setup
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getConf());
- this.vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
- try {
- for (final String key : this.memoryKeys) {
- MemoryHelper.validateKey(key);
- this.registerPersistentAggregator(key, MemoryAggregator.class);
- }
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- this.vertexProgram.setup(this);
- } else {
- if (this.vertexProgram.terminate(this)) { // terminate
- // write the memory to HDFS
- final MapMemory memory = new MapMemory(this);
- // a hack to get the last iteration memory values to stick
- this.vertexProgram.terminate(memory);
- final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
- if (null != outputLocation) {
- try {
- for (final String key : this.keys()) {
- final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + key), ObjectWritable.class, ObjectWritable.class);
- writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.get(key)));
- writer.close();
- }
- final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + Constants.HIDDEN_ITERATION), ObjectWritable.class, ObjectWritable.class);
- writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.getIteration()));
- writer.close();
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
- this.haltComputation();
- }
- }
- }
-
- @Override
- public int getIteration() {
- if (this.isMasterCompute) {
- final int temp = (int) this.getSuperstep();
- return temp == 0 ? temp : temp - 1;
- } else {
- return (int) this.worker.getSuperstep();
- }
- }
-
- @Override
- public long getRuntime() {
- return System.currentTimeMillis() - this.startTime;
- }
-
- @Override
- public Set<String> keys() {
- return this.memoryKeys.stream().filter(this::exists).collect(Collectors.toSet());
- }
-
- @Override
- public boolean exists(final String key) {
- final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
- return null != rule.getObject();
- }
-
- @Override
- public <R> R get(final String key) throws IllegalArgumentException {
- //this.checkKey(key);
- final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
- if (null == rule.getObject())
- throw Memory.Exceptions.memoryDoesNotExist(key);
- else
- return rule.getObject();
- }
-
- @Override
- public void set(final String key, Object value) {
- this.checkKeyValue(key, value);
- if (this.isMasterCompute)
- this.setAggregatedValue(key, new Rule(Rule.Operation.SET, value));
- else
- this.worker.aggregate(key, new Rule(Rule.Operation.SET, value));
- }
-
- @Override
- public void and(final String key, final boolean bool) {
- this.checkKeyValue(key, bool);
- if (this.isMasterCompute) { // only called on setup() and terminate()
- Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
- value = null == value ? bool : bool && value;
- this.setAggregatedValue(key, new Rule(Rule.Operation.AND, value));
- } else {
- this.worker.aggregate(key, new Rule(Rule.Operation.AND, bool));
- }
- }
-
- @Override
- public void or(final String key, final boolean bool) {
- this.checkKeyValue(key, bool);
- if (this.isMasterCompute) { // only called on setup() and terminate()
- Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
- value = null == value ? bool : bool || value;
- this.setAggregatedValue(key, new Rule(Rule.Operation.OR, value));
- } else {
- this.worker.aggregate(key, new Rule(Rule.Operation.OR, bool));
- }
- }
-
- @Override
- public void incr(final String key, final long delta) {
- this.checkKeyValue(key, delta);
- if (this.isMasterCompute) { // only called on setup() and terminate()
- Number value = this.<Rule>getAggregatedValue(key).<Number>getObject();
- value = null == value ? delta : value.longValue() + delta;
- this.setAggregatedValue(key, new Rule(Rule.Operation.INCR, value));
- } else {
- this.worker.aggregate(key, new Rule(Rule.Operation.INCR, delta));
- }
- }
-
- @Override
- public void write(final DataOutput output) {
- // no need to serialize the master compute as it gets its data from aggregators
- // is this true?
- }
-
- @Override
- public void readFields(final DataInput input) {
- // no need to serialize the master compute as it gets its data from aggregators
- // is this true?
- }
-
- @Override
- public String toString() {
- return StringFactory.memoryString(this);
- }
-
- private void checkKeyValue(final String key, final Object value) {
- if (!this.memoryKeys.contains(key))
- throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
- MemoryHelper.validateValue(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java
deleted file mode 100644
index 1494d51..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessageCombiner.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.combiner.Combiner;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMessageCombiner extends Combiner<ObjectWritable, ObjectWritable> implements ImmutableClassesGiraphConfigurable {
-
- private MessageCombiner messageCombiner;
- private ImmutableClassesGiraphConfiguration configuration;
-
- @Override
- public void combine(final ObjectWritable vertexIndex, final ObjectWritable originalMessage, final ObjectWritable messageToCombine) {
- originalMessage.set(originalMessage.isEmpty() ?
- messageToCombine.get() :
- this.messageCombiner.combine(originalMessage.get(), messageToCombine.get()));
- }
-
- @Override
- public ObjectWritable createInitialMessage() {
- return ObjectWritable.empty();
- }
-
- @Override
- public void setConf(final ImmutableClassesGiraphConfiguration configuration) {
- this.configuration = configuration;
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration);
- this.messageCombiner = (MessageCombiner) VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().get();
- }
-
- @Override
- public ImmutableClassesGiraphConfiguration getConf() {
- return this.configuration;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java
deleted file mode 100644
index 83b2943..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphMessenger.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMessenger<M> implements Messenger<M> {
-
- private GiraphComputeVertex giraphComputeVertex;
- private Iterator<ObjectWritable<M>> messages;
-
- public GiraphMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable<M>> messages) {
- this.giraphComputeVertex = giraphComputeVertex;
- this.messages = messages;
- }
-
- @Override
- public Iterator<M> receiveMessages() {
- return IteratorUtils.map(this.messages, ObjectWritable::get);
- }
-
- @Override
- public void sendMessage(final MessageScope messageScope, final M message) {
- if (messageScope instanceof MessageScope.Local) {
- final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
- final Traversal.Admin<Vertex, Edge> incidentTraversal = GiraphMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.giraphComputeVertex.getValue().get());
- final Direction direction = GiraphMessenger.getOppositeDirection(incidentTraversal);
- incidentTraversal.forEachRemaining(edge ->
- this.giraphComputeVertex.sendMessage(
- new ObjectWritable<>(edge.vertices(direction).next().id()),
- new ObjectWritable<>(localMessageScope.getEdgeFunction().apply(message, edge))));
- } else {
- final MessageScope.Global globalMessageScope = (MessageScope.Global) messageScope;
- globalMessageScope.vertices().forEach(vertex ->
- this.giraphComputeVertex.sendMessage(new ObjectWritable<>(vertex.id()), new ObjectWritable<>(message)));
- }
- }
-
- private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
- incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
- return (T) incidentTraversal;
- }
-
- private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
- final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
- return step.getDirection().opposite();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
deleted file mode 100644
index 85d2c2d..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.ImmutableMemory;
-import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
-
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphWorkerContext extends WorkerContext {
-
- private VertexProgramPool vertexProgramPool;
- private GiraphMemory memory;
-
- public GiraphWorkerContext() {
- // Giraph ReflectionUtils requires this to be public at minimum
- }
-
- public void preApplication() throws InstantiationException, IllegalAccessException {
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration());
- HadoopPools.initialize(apacheConfiguration);
- final VertexProgram vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
- this.memory = new GiraphMemory(this, vertexProgram);
- }
-
- public void postApplication() {
-
- }
-
- public void preSuperstep() {
- this.vertexProgramPool.workerIterationStart(new ImmutableMemory(this.memory));
- }
-
- public void postSuperstep() {
- this.vertexProgramPool.workerIterationEnd(new ImmutableMemory(this.memory));
- }
-
- public VertexProgramPool getVertexProgramPool() {
- return this.vertexProgramPool;
- }
-
- public GiraphMemory getMemory() {
- return this.memory;
- }
-
- public GiraphMessenger getMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable> messages) {
- return new GiraphMessenger(giraphComputeVertex, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/MemoryAggregator.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/MemoryAggregator.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/MemoryAggregator.java
deleted file mode 100644
index 8f11fb3..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/MemoryAggregator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.giraph.aggregators.Aggregator;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class MemoryAggregator implements Aggregator<Rule> {
-
- private Object currentObject;
- private Rule.Operation lastOperation = null;
-
- public MemoryAggregator() {
- this.currentObject = null;
- }
-
- @Override
- public Rule getAggregatedValue() {
- if (null == this.currentObject)
- return createInitialValue();
- else if (this.currentObject instanceof Long)
- return new Rule(Rule.Operation.INCR, this.currentObject);
- else
- return new Rule(null == this.lastOperation ? Rule.Operation.NO_OP : this.lastOperation, this.currentObject);
- }
-
- @Override
- public void setAggregatedValue(final Rule rule) {
- this.currentObject = rule.getObject();
- }
-
- @Override
- public void reset() {
- this.currentObject = null;
- }
-
- @Override
- public Rule createInitialValue() {
- return new Rule(Rule.Operation.NO_OP, null);
- }
-
- @Override
- public void aggregate(final Rule ruleWritable) {
- final Rule.Operation rule = ruleWritable.getOperation();
- final Object object = ruleWritable.getObject();
- if (rule != Rule.Operation.NO_OP)
- this.lastOperation = rule;
-
- if (null == this.currentObject || rule.equals(Rule.Operation.SET)) {
- this.currentObject = object;
- } else {
- if (rule.equals(Rule.Operation.INCR)) {
- this.currentObject = (Long) this.currentObject + (Long) object;
- } else if (rule.equals(Rule.Operation.AND)) {
- this.currentObject = (Boolean) this.currentObject && (Boolean) object;
- } else if (rule.equals(Rule.Operation.OR)) {
- this.currentObject = (Boolean) this.currentObject || (Boolean) object;
- } else if (rule.equals(Rule.Operation.NO_OP)) {
- if (object instanceof Boolean) { // only happens when NO_OP booleans are being propagated will this occur
- if (null == this.lastOperation) {
- // do nothing ... why?
- } else if (this.lastOperation.equals(Rule.Operation.AND)) {
- this.currentObject = (Boolean) this.currentObject && (Boolean) object;
- } else if (this.lastOperation.equals(Rule.Operation.OR)) {
- this.currentObject = (Boolean) this.currentObject || (Boolean) object;
- } else {
- throw new IllegalStateException("This state should not have occurred: " + ruleWritable);
- }
- }
- } else {
- throw new IllegalArgumentException("The provided rule is unknown: " + ruleWritable);
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexInputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexInputFormat.java
deleted file mode 100644
index 7dcaae7..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexInputFormat.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io;
-
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.giraph.io.VertexInputFormat;
-import org.apache.giraph.io.VertexReader;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexInputFormat extends VertexInputFormat {
-
- private InputFormat<NullWritable, VertexWritable> hadoopGraphInputFormat;
-
- @Override
- public List<InputSplit> getSplits(final JobContext context, final int minSplitCountHint) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- return this.hadoopGraphInputFormat.getSplits(context);
- }
-
- @Override
- public VertexReader createVertexReader(final InputSplit split, final TaskAttemptContext context) throws IOException {
- this.constructor(context.getConfiguration());
- try {
- return new GiraphVertexReader(this.hadoopGraphInputFormat.createRecordReader(split, context));
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-
- private final void constructor(final Configuration configuration) {
- if (null == this.hadoopGraphInputFormat) {
- this.hadoopGraphInputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class, InputFormat.class), configuration);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexOutputFormat.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexOutputFormat.java
deleted file mode 100644
index b2ad139..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexOutputFormat.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io;
-
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.giraph.io.VertexOutputFormat;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexOutputFormat extends VertexOutputFormat {
-
- private OutputFormat<NullWritable, VertexWritable> hadoopGraphOutputFormat;
-
- @Override
- public VertexWriter createVertexWriter(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- return new GiraphVertexWriter(this.hadoopGraphOutputFormat);
- }
-
- @Override
- public void checkOutputSpecs(final JobContext context) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- this.hadoopGraphOutputFormat.checkOutputSpecs(context);
- }
-
- @Override
- public OutputCommitter getOutputCommitter(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.constructor(context.getConfiguration());
- return this.hadoopGraphOutputFormat.getOutputCommitter(context);
- }
-
- private final void constructor(final Configuration configuration) {
- if (null == this.hadoopGraphOutputFormat) {
- this.hadoopGraphOutputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, OutputFormat.class, OutputFormat.class), configuration);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexReader.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexReader.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexReader.java
deleted file mode 100644
index c78c7c8..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexReader.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.io.VertexReader;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphComputeVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexReader extends VertexReader {
-
- private RecordReader<NullWritable, VertexWritable> recordReader;
-
- public GiraphVertexReader(final RecordReader<NullWritable, VertexWritable> recordReader) {
- this.recordReader = recordReader;
- }
-
- @Override
- public void initialize(final InputSplit inputSplit, final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordReader.initialize(inputSplit, context);
- }
-
- @Override
- public boolean nextVertex() throws IOException, InterruptedException {
- return this.recordReader.nextKeyValue();
- }
-
- @Override
- public Vertex getCurrentVertex() throws IOException, InterruptedException {
- return new GiraphComputeVertex(this.recordReader.getCurrentValue());
- }
-
- @Override
- public void close() throws IOException {
- this.recordReader.close();
- }
-
- @Override
- public float getProgress() throws IOException, InterruptedException {
- return this.recordReader.getProgress();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexWriter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexWriter.java
deleted file mode 100644
index c66c801..0000000
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/io/GiraphVertexWriter.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.io;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphComputeVertex;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexWriter extends VertexWriter {
- private final OutputFormat<NullWritable, VertexWritable> outputFormat;
- private RecordWriter<NullWritable, VertexWritable> recordWriter;
-
- public GiraphVertexWriter(final OutputFormat<NullWritable, VertexWritable> outputFormat) {
- this.outputFormat = outputFormat;
- }
-
- @Override
- public void initialize(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordWriter = this.outputFormat.getRecordWriter(context);
- }
-
- @Override
- public void close(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordWriter.close(context);
- }
-
- @Override
- public void writeVertex(final Vertex vertex) throws IOException, InterruptedException {
- this.recordWriter.write(NullWritable.get(), ((GiraphComputeVertex) vertex).getValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 3c3c9b7..ae4f910 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -23,7 +23,6 @@ import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer;
import org.apache.tinkerpop.gremlin.hadoop.structure.hdfs.HadoopEdgeIterator;
import org.apache.tinkerpop.gremlin.hadoop.structure.hdfs.HadoopVertexIterator;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
@@ -66,18 +65,18 @@ import java.util.stream.Stream;
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_knows_b__c_knows_bX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ // computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX",
@@ -85,18 +84,18 @@ import java.util.stream.Stream;
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_knows_b__c_knows_bX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals",
method = "g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX",
@@ -140,18 +139,18 @@ import java.util.stream.Stream;
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
method = "shouldNotAllowNullMemoryKeys",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
method = "shouldNotAllowSettingUndeclaredMemoryKeys",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest",
method = "shouldHaveConsistentMemoryVertexPropertiesAndExceptions",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {GiraphGraphComputer.class})
+ reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
+ //computers = {GiraphGraphComputer.class})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileTest$Traversals",
method = "g_V_out_out_profile_grateful",
@@ -189,15 +188,15 @@ public final class HadoopGraph implements Graph {
@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) {
- if (graphComputerClass.equals(GiraphGraphComputer.class))
- return (C) new GiraphGraphComputer(this);
- else {
+ //if (graphComputerClass.equals(GiraphGraphComputer.class))
+ // return (C) new GiraphGraphComputer(this);
+ //else {
try {
return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
} catch (final Exception e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
- }
+ //}
//else if (graphComputerClass.equals(SparkGraphComputer.class))
// return (C) new SparkGraphComputer(this);
//else
@@ -206,7 +205,8 @@ public final class HadoopGraph implements Graph {
@Override
public GraphComputer compute() {
- return this.compute(GiraphGraphComputer.class);
+ return null;
+ //return this.compute(GiraphGraphComputer.class);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
index 27b84f0..ba0e75e 100644
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
+++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java
@@ -19,8 +19,6 @@
package org.apache.tinkerpop.gremlin.hadoop;
import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.TestHelper;
@@ -112,16 +110,16 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
/// giraph configuration
- put(GiraphConstants.MIN_WORKERS, 1);
- put(GiraphConstants.MAX_WORKERS, 1);
- put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
- put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
- put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
- put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
- put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
- put("mapred.reduce.tasks", 4);
+ //put(GiraphConstants.MIN_WORKERS, 1);
+ //put(GiraphConstants.MAX_WORKERS, 1);
+ //put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
+ //put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
+ //put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ //put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ //put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
+ //put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
+ //put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
+ //put("mapred.reduce.tasks", 4);
//put("giraph.vertexOutputFormatThreadSafe", false);
//put("giraph.numOutputThreads", 3);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputerProcessIntegrateTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputerProcessIntegrateTest.java
deleted file mode 100644
index 9b62137..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphGraphComputerProcessIntegrateTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(ProcessComputerSuite.class)
-@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
-public class GiraphGraphComputerProcessIntegrateTest {
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/HadoopGiraphGraphProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/HadoopGiraphGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/HadoopGiraphGraphProvider.java
deleted file mode 100644
index ae3e484..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/HadoopGiraphGraphProvider.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph;
-
-import org.apache.tinkerpop.gremlin.GraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@GraphProvider.Descriptor(computer = GiraphGraphComputer.class)
-public final class HadoopGiraphGraphProvider extends HadoopGraphProvider {
-
- public GraphTraversalSource traversal(final Graph graph) {
- return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(GiraphGraphComputer.class)).create(graph);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java
deleted file mode 100644
index d6aae67..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.groovy;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.HadoopGiraphGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.process.GroovyProcessComputerSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(GroovyProcessComputerSuite.class)
-@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
-public class GiraphGraphComputerGroovyProcessIntegrateTest {
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphHadoopGremlinPluginIntegrateTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
deleted file mode 100644
index 8463b31..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/groovy/GiraphHadoopGremlinPluginIntegrateTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.groovy;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopPluginSuite;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.HadoopGiraphGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(HadoopPluginSuite.class)
-@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
-public class GiraphHadoopGremlinPluginIntegrateTest {
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7a4411c..d822651 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,6 +117,7 @@ limitations under the License.
<module>gremlin-console</module>
<module>gremlin-server</module>
<module>spark-gremlin</module>
+ <module>giraph-gremlin</module>
</modules>
<scm>
<connection>scm:git:git@git-wip-us.apache.org:repos/asf/incubator-tinkerpop.git</connection>
[07/18] incubator-tinkerpop git commit: pulled out giraph-gremlin
from hadoop-gremlin. HadoopGremlin builds nicely. Giraph too. Spark is still
causing problems.
Posted by ok...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 645c97f..bf33ae5 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -132,12 +132,6 @@
<scope>test</scope>
</dependency>
</dependencies>
- <repositories>
- <repository>
- <id>hyracks-releases</id>
- <url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public-releases/</url>
- </repository>
- </repositories>
<build>
<directory>${basedir}/target</directory>
<finalName>${project.artifactId}-${project.version}</finalName>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
index 333636e..ebda863 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
@@ -45,7 +45,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
[14/18] incubator-tinkerpop git commit: HadoopProvider fix.
Posted by ok...@apache.org.
HadoopProvider fix.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/65d5a613
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/65d5a613
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/65d5a613
Branch: refs/heads/master
Commit: 65d5a613cb62844791600bbd7f571317814e9c7f
Parents: ffb42a1
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 8 18:04:39 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 8 18:04:39 2015 -0600
----------------------------------------------------------------------
pom.xml | 4 ++--
.../gremlin/spark/process/computer/io/OutputRDDTest.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/65d5a613/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d822651..23edec1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -112,12 +112,12 @@ limitations under the License.
<module>gremlin-groovy-test</module>
<module>tinkergraph-gremlin</module>
<module>hadoop-gremlin</module>
+ <module>spark-gremlin</module>
+ <module>giraph-gremlin</module>
<module>neo4j-gremlin</module>
<module>gremlin-driver</module>
<module>gremlin-console</module>
<module>gremlin-server</module>
- <module>spark-gremlin</module>
- <module>giraph-gremlin</module>
</modules>
<scm>
<connection>scm:git:git@git-wip-us.apache.org:repos/asf/incubator-tinkerpop.git</connection>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/65d5a613/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java
index 4ded9bc..6a2c24c 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/OutputRDDTest.java
@@ -27,7 +27,7 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.spark.process.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.spark.process.computer.HadoopSparkGraphProvider;
import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
@@ -44,7 +44,7 @@ public class OutputRDDTest {
configuration.setProperty("spark.master", "local[4]");
configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, HadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
+ configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, HadoopSparkGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName());
configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
[02/18] incubator-tinkerpop git commit: Spark is now isolated into
spark-gremlin package. Tests are passing. A few hacks here and there just to
get things building. However,
it was pretty easy to split apart -- which is pomising.
Posted by ok...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java
deleted file mode 100644
index cc07bac..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkGraphComputerGroovyProcessIntegrateTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.groovy;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.HadoopSparkGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.process.GroovyProcessComputerSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(GroovyProcessComputerSuite.class)
-@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
-public class SparkGraphComputerGroovyProcessIntegrateTest {
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkHadoopGremlinPluginTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkHadoopGremlinPluginTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkHadoopGremlinPluginTest.java
deleted file mode 100644
index 3d83142..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/groovy/SparkHadoopGremlinPluginTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.groovy;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopPluginSuite;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.HadoopSparkGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(HadoopPluginSuite.class)
-@GraphProviderClass(provider = HadoopSparkGraphProvider.class, graph = HadoopGraph.class)
-public class SparkHadoopGremlinPluginTest {
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleInputRDD.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleInputRDD.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleInputRDD.java
deleted file mode 100644
index d456808..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleInputRDD.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.structure.T;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-import scala.Tuple2;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ExampleInputRDD implements InputRDD {
-
- @Override
- public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
- final List<Vertex> list = new ArrayList<>();
- list.add(StarGraph.open().addVertex(T.id, 1l, T.label,"person","age", 29));
- list.add(StarGraph.open().addVertex(T.id, 2l, T.label,"person","age", 27));
- list.add(StarGraph.open().addVertex(T.id, 4l, T.label,"person","age", 32));
- list.add(StarGraph.open().addVertex(T.id, 6l, T.label,"person","age", 35));
- return sparkContext.parallelize(list).mapToPair(vertex -> new Tuple2<>(vertex.id(), new VertexWritable(vertex)));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleOutputRDD.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleOutputRDD.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleOutputRDD.java
deleted file mode 100644
index 45ed114..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/ExampleOutputRDD.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-
-import java.util.Iterator;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class ExampleOutputRDD implements OutputRDD {
- @Override
- public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object, VertexWritable> graphRDD) {
- int totalAge = 0;
- final Iterator<VertexWritable> iterator = graphRDD.values().toLocalIterator();
- while (iterator.hasNext()) {
- final Vertex vertex = iterator.next().get();
- if (vertex.label().equals("person"))
- totalAge = totalAge + vertex.<Integer>value("age");
- }
- assertEquals(123, totalAge);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputOutputRDDTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputOutputRDDTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputOutputRDDTest.java
deleted file mode 100644
index ea1cb87..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputOutputRDDTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
-import org.junit.Test;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class InputOutputRDDTest {
-
- @Test
- public void shouldReadFromWriteToArbitraryRDD() throws Exception {
- final Configuration configuration = new BaseConfiguration();
- configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
- configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- ////////
- Graph graph = GraphFactory.open(configuration);
- graph.compute(SparkGraphComputer.class)
- .result(GraphComputer.ResultGraph.NEW)
- .persist(GraphComputer.Persist.EDGES)
- .program(TraversalVertexProgram.build()
- .traversal(GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)),
- "gremlin-groovy",
- "g.V()").create(graph)).submit().get();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDDTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDDTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDDTest.java
deleted file mode 100644
index c600327..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/InputRDDTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class InputRDDTest {
-
- @Test
- public void shouldReadFromArbitraryRDD() {
- final Configuration configuration = new BaseConfiguration();
- configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
- configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- ////////
- Graph graph = GraphFactory.open(configuration);
- assertEquals(Double.valueOf(123.0d), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
- assertEquals(Long.valueOf(4l), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().count().next());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDDTest.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDDTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDDTest.java
deleted file mode 100644
index deea6ab..0000000
--- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/io/OutputRDDTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.io;
-
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
-import org.junit.Test;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class OutputRDDTest {
-
- @Test
- public void shouldWriteToArbitraryRDD() throws Exception {
- final Configuration configuration = new BaseConfiguration();
- configuration.setProperty("spark.master", "local[4]");
- configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
- configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, HadoopGraphProvider.PATHS.get("tinkerpop-modern.kryo"));
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, GryoInputFormat.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, ExampleOutputRDD.class.getCanonicalName());
- configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
- configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- ////////
- Graph graph = GraphFactory.open(configuration);
- graph.compute(SparkGraphComputer.class)
- .result(GraphComputer.ResultGraph.NEW)
- .persist(GraphComputer.Persist.EDGES)
- .program(TraversalVertexProgram.build()
- .traversal(GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(SparkGraphComputer.class)),
- "gremlin-groovy",
- "g.V()").create(graph)).submit().get();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 769f84f..7a4411c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,6 +116,7 @@ limitations under the License.
<module>gremlin-driver</module>
<module>gremlin-console</module>
<module>gremlin-server</module>
+ <module>spark-gremlin</module>
</modules>
<scm>
<connection>scm:git:git@git-wip-us.apache.org:repos/asf/incubator-tinkerpop.git</connection>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
new file mode 100644
index 0000000..277ce11
--- /dev/null
+++ b/spark-gremlin/pom.xml
@@ -0,0 +1,249 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkerpop</artifactId>
+ <version>3.1.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>spark-gremlin</artifactId>
+ <name>Apache TinkerPop :: Spark Gremlin</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>hadoop-gremlin</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- SPARK GRAPH COMPUTER -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>1.2.1</version>
+ <exclusions>
+ <!-- self conflicts -->
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <!-- gremlin-core conflicts -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </exclusion>
+ <!-- gremlin-groovy conflicts -->
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <!-- hadoop conflicts -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <!-- giraph conflicts -->
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <!-- lgpl conflicts -->
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>findbugs</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- consistent dependencies -->
+<!-- <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.10.3</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.5.13.Final</version>
+ </dependency>
+-->
+ <!-- TEST -->
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-test</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy-test</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkergraph-gremlin</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <repositories>
+ <repository>
+ <id>hyracks-releases</id>
+ <url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public-releases/</url>
+ </repository>
+ </repositories>
+ <build>
+ <directory>${basedir}/target</directory>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <id>build-detached-assemblies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/assembly/standalone.xml</descriptor>
+ <descriptor>src/assembly/hadoop-job.xml</descriptor>
+ </descriptors>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.gmavenplus</groupId>
+ <artifactId>gmavenplus-plugin</artifactId>
+ <version>1.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>addSources</goal>
+ <goal>addTestSources</goal>
+ <goal>generateStubs</goal>
+ <goal>compile</goal>
+ <goal>testGenerateStubs</goal>
+ <goal>testCompile</goal>
+ <goal>removeStubs</goal>
+ <goal>removeTestStubs</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <invokeDynamic>true</invokeDynamic>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.5</version>
+ <configuration>
+ <archive>
+ <manifestEntries>
+ <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-core:1.2.1
+ </Gremlin-Plugin-Dependencies>
+ <!-- deletes the servlet-api jar from the path after install - causes conflicts -->
+ <Gremlin-Plugin-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
+ </Gremlin-Plugin-Paths>
+ <Gremlin-Lib-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
+ </Gremlin-Lib-Paths>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- log4j.configuration>log4j-core.properties</log4j.configuration> -->
+ <!--<argLine>-Xmx2048M</argLine>-->
+ <excludes>
+ <exclude>**/*IntegrateTest.java</exclude>
+ <exclude>**/*PerformanceTest.java</exclude>
+ <!-- this is technically a member of the integration test suite -->
+ <exclude>**/HadoopGremlinPluginTest.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
new file mode 100644
index 0000000..0b04300
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/groovy/plugin/SparkGremlinPlugin.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.groovy.plugin;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class SparkGremlinPlugin {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/RuleAccumulator.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/RuleAccumulator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/RuleAccumulator.java
new file mode 100644
index 0000000..bebd283
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/RuleAccumulator.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.spark.AccumulatorParam;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class RuleAccumulator implements AccumulatorParam<Rule> {
+
+ @Override
+ public Rule addAccumulator(final Rule a, final Rule b) {
+ if (a.getOperation().equals(Rule.Operation.NO_OP))
+ return b;
+ if (b.getOperation().equals(Rule.Operation.NO_OP))
+ return a;
+ else
+ return new Rule(b.getOperation(), b.getOperation().compute(a.getObject(), b.getObject()));
+ }
+
+ @Override
+ public Rule addInPlace(final Rule a, final Rule b) {
+ if (a.getOperation().equals(Rule.Operation.NO_OP))
+ return b;
+ if (b.getOperation().equals(Rule.Operation.NO_OP))
+ return a;
+ else
+ return new Rule(b.getOperation(), b.getOperation().compute(a.getObject(), b.getObject()));
+ }
+
+ @Override
+ public Rule zero(final Rule rule) {
+ return new Rule(Rule.Operation.NO_OP, null);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
new file mode 100644
index 0000000..288538f
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkExecutor.java
@@ -0,0 +1,202 @@
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import com.google.common.base.Optional;
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.MessagePayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.Payload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewOutgoingPayload;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewPayload;
+import org.apache.tinkerpop.gremlin.structure.util.Attachable;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
+import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import scala.Tuple2;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkExecutor {
+
+ private static final String[] EMPTY_ARRAY = new String[0];
+
+ private SparkExecutor() {
+ }
+
+ ////////////////////
+ // VERTEX PROGRAM //
+ ////////////////////
+
+ public static <M> JavaPairRDD<Object, ViewIncomingPayload<M>> executeVertexProgramIteration(
+ final JavaPairRDD<Object, VertexWritable> graphRDD,
+ final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD,
+ final SparkMemory memory,
+ final Configuration apacheConfiguration) {
+
+ final JavaPairRDD<Object, ViewOutgoingPayload<M>> viewOutgoingRDD = (((null == viewIncomingRDD) ?
+ graphRDD.mapValues(vertexWritable -> new Tuple2<>(vertexWritable, Optional.<ViewIncomingPayload<M>>absent())) : // first iteration will not have any views or messages
+ graphRDD.leftOuterJoin(viewIncomingRDD)) // every other iteration may have views and messages
+ // for each partition of vertices
+ .mapPartitionsToPair(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
+ final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
+ final Set<String> elementComputeKeys = workerVertexProgram.getElementComputeKeys(); // the compute keys as a set
+ final String[] elementComputeKeysArray = elementComputeKeys.size() == 0 ? EMPTY_ARRAY : elementComputeKeys.toArray(new String[elementComputeKeys.size()]); // the compute keys as an array
+ final SparkMessenger<M> messenger = new SparkMessenger<>();
+ workerVertexProgram.workerIterationStart(memory.asImmutable()); // start the worker
+ return () -> IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
+ final StarGraph.StarVertex vertex = vertexViewIncoming._2()._1().get(); // get the vertex from the vertex writable
+ // drop any compute properties that are cached in memory
+ if (elementComputeKeysArray.length > 0)
+ vertex.dropVertexProperties(elementComputeKeysArray);
+ final boolean hasViewAndMessages = vertexViewIncoming._2()._2().isPresent(); // if this is the first iteration, then there are no views or messages
+ final List<DetachedVertexProperty<Object>> previousView = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getView() : Collections.emptyList();
+ final List<M> incomingMessages = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getIncomingMessages() : Collections.emptyList();
+ previousView.forEach(property -> property.attach(Attachable.Method.create(vertex))); // attach the view to the vertex
+ ///
+ messenger.setVertexAndIncomingMessages(vertex, incomingMessages); // set the messenger with the incoming messages
+ workerVertexProgram.execute(ComputerGraph.vertexProgram(vertex, workerVertexProgram), messenger, memory); // execute the vertex program on this vertex for this iteration
+ ///
+ final List<DetachedVertexProperty<Object>> nextView = elementComputeKeysArray.length == 0 ? // not all vertex programs have compute keys
+ Collections.emptyList() :
+ IteratorUtils.list(IteratorUtils.map(vertex.properties(elementComputeKeysArray), property -> DetachedFactory.detach(property, true)));
+ final List<Tuple2<Object, M>> outgoingMessages = messenger.getOutgoingMessages(); // get the outgoing messages
+ if (!partitionIterator.hasNext())
+ workerVertexProgram.workerIterationEnd(memory.asImmutable()); // if no more vertices in the partition, end the worker's iteration
+ return new Tuple2<>(vertex.id(), new ViewOutgoingPayload<>(nextView, outgoingMessages));
+ });
+ })).setName("viewOutgoingRDD");
+
+ // "message pass" by reducing on the vertex object id of the view and message payloads
+ final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().orElse(null);
+ final JavaPairRDD<Object, ViewIncomingPayload<M>> newViewIncomingRDD = viewOutgoingRDD
+ .flatMapToPair(tuple -> () -> IteratorUtils.<Tuple2<Object, Payload>>concat(
+ IteratorUtils.of(new Tuple2<>(tuple._1(), tuple._2().getView())), // emit the view payload
+ IteratorUtils.map(tuple._2().getOutgoingMessages().iterator(), message -> new Tuple2<>(message._1(), new MessagePayload<>(message._2()))))) // emit the outgoing message payloads one by one
+ .reduceByKey((a, b) -> { // reduce the view and outgoing messages into a single payload object representing the new view and incoming messages for a vertex
+ if (a instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) a).mergePayload(b, messageCombiner);
+ return a;
+ } else if (b instanceof ViewIncomingPayload) {
+ ((ViewIncomingPayload<M>) b).mergePayload(a, messageCombiner);
+ return b;
+ } else {
+ final ViewIncomingPayload<M> c = new ViewIncomingPayload<>(messageCombiner);
+ c.mergePayload(a, messageCombiner);
+ c.mergePayload(b, messageCombiner);
+ return c;
+ }
+ })
+ .filter(payload -> !(payload._2() instanceof MessagePayload)) // this happens if there is a message to a vertex that does not exist
+ .filter(payload -> !((payload._2() instanceof ViewIncomingPayload) && !((ViewIncomingPayload<M>) payload._2()).hasView())) // this happens if there are many messages to a vertex that does not exist
+ .mapValues(payload -> payload instanceof ViewIncomingPayload ?
+ (ViewIncomingPayload<M>) payload : // this happens if there is a vertex with incoming messages
+ new ViewIncomingPayload<>((ViewPayload) payload)); // this happens if there is a vertex with no incoming messages
+
+ newViewIncomingRDD.setName("viewIncomingRDD")
+ .foreachPartition(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
+ }); // need to complete a task so its BSP and the memory for this iteration is updated
+ return newViewIncomingRDD;
+ }
+
+ /////////////////
+ // MAP REDUCE //
+ ////////////////
+
+ public static <M> JavaPairRDD<Object, VertexWritable> prepareGraphRDDForMapReduce(final JavaPairRDD<Object, VertexWritable> graphRDD, final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD, final String[] elementComputeKeys) {
+ return (null == viewIncomingRDD) ? // there was no vertex program
+ graphRDD.mapValues(vertexWritable -> {
+ vertexWritable.get().dropEdges();
+ return vertexWritable;
+ }) :
+ graphRDD.leftOuterJoin(viewIncomingRDD)
+ .mapValues(tuple -> {
+ final StarGraph.StarVertex vertex = tuple._1().get();
+ vertex.dropEdges();
+ vertex.dropVertexProperties(elementComputeKeys);
+ final List<DetachedVertexProperty<Object>> view = tuple._2().isPresent() ? tuple._2().get().getView() : Collections.emptyList();
+ view.forEach(property -> property.attach(Attachable.Method.create(vertex)));
+ return tuple._1();
+ });
+ }
+
+ public static <K, V> JavaPairRDD<K, V> executeMap(final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce, final Configuration apacheConfiguration) {
+ JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
+ final MapReduce<K, V, ?, ?, ?> workerMapReduce = MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
+ workerMapReduce.workerStart(MapReduce.Stage.MAP);
+ final SparkMapEmitter<K, V> mapEmitter = new SparkMapEmitter<>();
+ return () -> IteratorUtils.flatMap(partitionIterator, vertexWritable -> {
+ workerMapReduce.map(ComputerGraph.mapReduce(vertexWritable._2().get()), mapEmitter);
+ if (!partitionIterator.hasNext())
+ workerMapReduce.workerEnd(MapReduce.Stage.MAP);
+ return mapEmitter.getEmissions();
+ });
+ });
+ if (mapReduce.getMapKeySort().isPresent())
+ mapRDD = mapRDD.sortByKey(mapReduce.getMapKeySort().get());
+ return mapRDD;
+ }
+
+ // TODO: public static executeCombine() is this necessary? YES --- we groupByKey in reduce() where we want to combine first.
+
+ public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce(final JavaPairRDD<K, V> mapRDD, final MapReduce<K, V, OK, OV, ?> mapReduce, final Configuration apacheConfiguration) {
+ JavaPairRDD<OK, OV> reduceRDD = mapRDD.groupByKey().mapPartitionsToPair(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
+ final MapReduce<K, V, OK, OV, ?> workerMapReduce = MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
+ workerMapReduce.workerStart(MapReduce.Stage.REDUCE);
+ final SparkReduceEmitter<OK, OV> reduceEmitter = new SparkReduceEmitter<>();
+ return () -> IteratorUtils.flatMap(partitionIterator, keyValue -> {
+ workerMapReduce.reduce(keyValue._1(), keyValue._2().iterator(), reduceEmitter);
+ if (!partitionIterator.hasNext())
+ workerMapReduce.workerEnd(MapReduce.Stage.REDUCE);
+ return reduceEmitter.getEmissions();
+ });
+ });
+ if (mapReduce.getReduceKeySort().isPresent())
+ reduceRDD = reduceRDD.sortByKey(mapReduce.getReduceKeySort().get());
+ return reduceRDD;
+ }
+
+ ///////////////////
+ // Input/Output //
+ //////////////////
+
+ public static void saveMapReduceRDD(final JavaPairRDD<Object, Object> mapReduceRDD, final MapReduce mapReduce, final Memory.Admin memory, final org.apache.hadoop.conf.Configuration hadoopConfiguration) {
+ final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
+ if (null != outputLocation) {
+ // map back to a Hadoop stream for output
+ mapReduceRDD.mapToPair(keyValue -> new Tuple2<>(new ObjectWritable<>(keyValue._1()), new ObjectWritable<>(keyValue._2()))).saveAsNewAPIHadoopFile(outputLocation + "/" + mapReduce.getMemoryKey(),
+ ObjectWritable.class,
+ ObjectWritable.class,
+ SequenceFileOutputFormat.class, hadoopConfiguration);
+ // TODO: mapReduce.addResultToMemory(memory, mapReduceRDD.map(tuple -> new KeyValue<>(tuple._1(), tuple._2())).collect().iterator());
+ try {
+ mapReduce.addResultToMemory(memory, new ObjectWritableIterator(hadoopConfiguration, new Path(outputLocation + "/" + mapReduce.getMemoryKey())));
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
new file mode 100644
index 0000000..3aa4383
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.commons.configuration.ConfigurationUtils;
+import org.apache.commons.configuration.FileConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
+import org.apache.tinkerpop.gremlin.spark.process.computer.io.InputFormatRDD;
+import org.apache.tinkerpop.gremlin.spark.process.computer.io.InputRDD;
+import org.apache.tinkerpop.gremlin.spark.process.computer.io.OutputFormatRDD;
+import org.apache.tinkerpop.gremlin.spark.process.computer.io.OutputRDD;
+import org.apache.tinkerpop.gremlin.spark.process.computer.payload.ViewIncomingPayload;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.HadoopHelper;
+import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.stream.Stream;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
+
+ public SparkGraphComputer(final HadoopGraph hadoopGraph) {
+ super(hadoopGraph);
+ }
+
+ @Override
+ public Future<ComputerResult> submit() {
+ super.validateStatePriorToExecution();
+ // apache and hadoop configurations that are used throughout the graph computer computation
+ final org.apache.commons.configuration.Configuration apacheConfiguration = new HadoopConfiguration(this.hadoopGraph.configuration());
+ apacheConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(GraphComputer.Persist.EDGES));
+ final Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(apacheConfiguration);
+ if (FileInputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
+ try {
+ final String inputLocation = FileSystem.get(hadoopConfiguration).getFileStatus(new Path(hadoopConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION))).getPath().toString();
+ apacheConfiguration.setProperty(Constants.MAPRED_INPUT_DIR, inputLocation);
+ hadoopConfiguration.set(Constants.MAPRED_INPUT_DIR, inputLocation);
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+
+ // create the completable future
+ return CompletableFuture.<ComputerResult>supplyAsync(() -> {
+ final long startTime = System.currentTimeMillis();
+ SparkMemory memory = null;
+ // delete output location
+ final String outputLocation = hadoopConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
+ if (null != outputLocation) {
+ try {
+ FileSystem.get(hadoopConfiguration).delete(new Path(outputLocation), true);
+ } catch (final IOException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ // wire up a spark context
+ final SparkConf sparkConfiguration = new SparkConf();
+ sparkConfiguration.setAppName(Constants.GREMLIN_HADOOP_SPARK_JOB_PREFIX + (null == this.vertexProgram ? "No VertexProgram" : this.vertexProgram) + "[" + this.mapReducers + "]");
+ /*final List<Class> classes = new ArrayList<>();
+ classes.addAll(IOClasses.getGryoClasses(GryoMapper.build().create()));
+ classes.addAll(IOClasses.getSharedHadoopClasses());
+ classes.add(ViewPayload.class);
+ classes.add(MessagePayload.class);
+ classes.add(ViewIncomingPayload.class);
+ classes.add(ViewOutgoingPayload.class);
+ sparkConfiguration.registerKryoClasses(classes.toArray(new Class[classes.size()]));*/ // TODO: fix for user submitted jars in Spark 1.3.0
+
+ // create the spark configuration from the graph computer configuration
+ hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
+ // execute the vertex program and map reducers and if there is a failure, auto-close the spark context
+ try (final JavaSparkContext sparkContext = new JavaSparkContext(sparkConfiguration)) {
+ // add the project jars to the cluster
+ this.loadJars(sparkContext, hadoopConfiguration);
+ // create a message-passing friendly rdd from the input rdd
+ final JavaPairRDD<Object, VertexWritable> graphRDD;
+ try {
+ graphRDD = hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_RDD, InputFormatRDD.class, InputRDD.class)
+ .newInstance()
+ .readGraphRDD(apacheConfiguration, sparkContext)
+ .setName("graphRDD")
+ .cache();
+ } catch (final InstantiationException | IllegalAccessException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ JavaPairRDD<Object, ViewIncomingPayload<Object>> viewIncomingRDD = null;
+
+ ////////////////////////////////
+ // process the vertex program //
+ ////////////////////////////////
+ if (null != this.vertexProgram) {
+ // set up the vertex program and wire up configurations
+ memory = new SparkMemory(this.vertexProgram, this.mapReducers, sparkContext);
+ this.vertexProgram.setup(memory);
+ memory.broadcastMemory(sparkContext);
+ final HadoopConfiguration vertexProgramConfiguration = new HadoopConfiguration();
+ this.vertexProgram.storeState(vertexProgramConfiguration);
+ ConfigurationUtils.copy(vertexProgramConfiguration, apacheConfiguration);
+ ConfUtil.mergeApacheIntoHadoopConfiguration(vertexProgramConfiguration, hadoopConfiguration);
+
+ // execute the vertex program
+ while (true) {
+ memory.setInTask(true);
+ viewIncomingRDD = SparkExecutor.executeVertexProgramIteration(graphRDD, viewIncomingRDD, memory, vertexProgramConfiguration);
+ memory.setInTask(false);
+ if (this.vertexProgram.terminate(memory))
+ break;
+ else {
+ memory.incrIteration();
+ memory.broadcastMemory(sparkContext);
+ }
+ }
+ // write the graph rdd using the output rdd
+ if (!this.persist.equals(GraphComputer.Persist.NOTHING)) {
+ try {
+ hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_RDD, OutputFormatRDD.class, OutputRDD.class)
+ .newInstance()
+ .writeGraphRDD(apacheConfiguration, graphRDD);
+ } catch (final InstantiationException | IllegalAccessException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ }
+
+ final Memory.Admin finalMemory = null == memory ? new MapMemory() : new MapMemory(memory);
+
+ //////////////////////////////
+ // process the map reducers //
+ //////////////////////////////
+ if (!this.mapReducers.isEmpty()) {
+ final String[] elementComputeKeys = this.vertexProgram == null ? new String[0] : this.vertexProgram.getElementComputeKeys().toArray(new String[this.vertexProgram.getElementComputeKeys().size()]);
+ final JavaPairRDD<Object, VertexWritable> mapReduceGraphRDD = SparkExecutor.prepareGraphRDDForMapReduce(graphRDD, viewIncomingRDD, elementComputeKeys).setName("mapReduceGraphRDD").cache();
+ for (final MapReduce mapReduce : this.mapReducers) {
+ // execute the map reduce job
+ final HadoopConfiguration newApacheConfiguration = new HadoopConfiguration(apacheConfiguration);
+ mapReduce.storeState(newApacheConfiguration);
+ // map
+ final JavaPairRDD mapRDD = SparkExecutor.executeMap((JavaPairRDD) mapReduceGraphRDD, mapReduce, newApacheConfiguration).setName("mapRDD");
+ // combine TODO: is this really needed
+ // reduce
+ final JavaPairRDD reduceRDD = (mapReduce.doStage(MapReduce.Stage.REDUCE)) ? SparkExecutor.executeReduce(mapRDD, mapReduce, newApacheConfiguration).setName("reduceRDD") : null;
+ // write the map reduce output back to disk (memory)
+ SparkExecutor.saveMapReduceRDD(null == reduceRDD ? mapRDD : reduceRDD, mapReduce, finalMemory, hadoopConfiguration);
+ }
+ }
+ // update runtime and return the newly computed graph
+ finalMemory.setRuntime(System.currentTimeMillis() - startTime);
+ return new DefaultComputerResult(HadoopHelper.getOutputGraph(this.hadoopGraph, this.resultGraph, this.persist), finalMemory.asImmutable());
+ }
+ });
+ }
+
+ /////////////////
+
+ private void loadJars(final JavaSparkContext sparkContext, final Configuration hadoopConfiguration) {
+ if (hadoopConfiguration.getBoolean(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true)) {
+ final String hadoopGremlinLocalLibs = System.getenv(Constants.HADOOP_GREMLIN_LIBS);
+ if (null == hadoopGremlinLocalLibs)
+ this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
+ else {
+ final String[] paths = hadoopGremlinLocalLibs.split(":");
+ for (final String path : paths) {
+ final File file = new File(path);
+ if (file.exists())
+ Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> sparkContext.addJar(f.getAbsolutePath()));
+ else
+ this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
+ }
+ }
+ }
+ }
+
+ public static void main(final String[] args) throws Exception {
+ final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
+ new SparkGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMapEmitter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMapEmitter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMapEmitter.java
new file mode 100644
index 0000000..cf31249
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMapEmitter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import scala.Tuple2;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkMapEmitter<K, V> implements MapReduce.MapEmitter<K, V> {
+
+ private List<Tuple2<K, V>> emissions = new ArrayList<>();
+
+ @Override
+ public void emit(final K key, final V value) {
+ this.emissions.add(new Tuple2<>(key, value));
+ }
+
+ public Iterator<Tuple2<K, V>> getEmissions() {
+ final Iterator<Tuple2<K,V>> iterator = this.emissions.iterator();
+ this.emissions = new ArrayList<>();
+ return iterator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
new file mode 100644
index 0000000..dbfadcc
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.spark.Accumulator;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkMemory implements Memory.Admin, Serializable {
+
+ public final Set<String> memoryKeys = new HashSet<>();
+ private final AtomicInteger iteration = new AtomicInteger(0); // do these need to be atomics?
+ private final AtomicLong runtime = new AtomicLong(0l);
+ private final Map<String, Accumulator<Rule>> memory = new HashMap<>();
+ private Broadcast<Map<String, Object>> broadcast;
+ private boolean inTask = false;
+
+ public SparkMemory(final VertexProgram<?> vertexProgram, final Set<MapReduce> mapReducers, final JavaSparkContext sparkContext) {
+ if (null != vertexProgram) {
+ for (final String key : vertexProgram.getMemoryComputeKeys()) {
+ MemoryHelper.validateKey(key);
+ this.memoryKeys.add(key);
+ }
+ }
+ for (final MapReduce mapReduce : mapReducers) {
+ this.memoryKeys.add(mapReduce.getMemoryKey());
+ }
+ for (final String key : this.memoryKeys) {
+ this.memory.put(key, sparkContext.accumulator(new Rule(Rule.Operation.NO_OP, null), key, new RuleAccumulator()));
+ }
+ this.broadcast = sparkContext.broadcast(new HashMap<>());
+ }
+
+ @Override
+ public Set<String> keys() {
+ if (this.inTask)
+ return this.broadcast.getValue().keySet();
+ else {
+ final Set<String> trueKeys = new HashSet<>();
+ this.memory.forEach((key, value) -> {
+ if (value.value().getObject() != null)
+ trueKeys.add(key);
+ });
+ return Collections.unmodifiableSet(trueKeys);
+ }
+ }
+
+ @Override
+ public void incrIteration() {
+ this.iteration.getAndIncrement();
+ }
+
+ @Override
+ public void setIteration(final int iteration) {
+ this.iteration.set(iteration);
+ }
+
+ @Override
+ public int getIteration() {
+ return this.iteration.get();
+ }
+
+ @Override
+ public void setRuntime(final long runTime) {
+ this.runtime.set(runTime);
+ }
+
+ @Override
+ public long getRuntime() {
+ return this.runtime.get();
+ }
+
+ @Override
+ public <R> R get(final String key) throws IllegalArgumentException {
+ final R r = this.getValue(key);
+ if (null == r)
+ throw Memory.Exceptions.memoryDoesNotExist(key);
+ else
+ return r;
+ }
+
+ @Override
+ public void incr(final String key, final long delta) {
+ checkKeyValue(key, delta);
+ if (this.inTask)
+ this.memory.get(key).add(new Rule(Rule.Operation.INCR, delta));
+ else
+ this.memory.get(key).setValue(new Rule(Rule.Operation.INCR, this.<Long>getValue(key) + delta));
+ }
+
+ @Override
+ public void and(final String key, final boolean bool) {
+ checkKeyValue(key, bool);
+ if (this.inTask)
+ this.memory.get(key).add(new Rule(Rule.Operation.AND, bool));
+ else
+ this.memory.get(key).setValue(new Rule(Rule.Operation.AND, this.<Boolean>getValue(key) && bool));
+ }
+
+ @Override
+ public void or(final String key, final boolean bool) {
+ checkKeyValue(key, bool);
+ if (this.inTask)
+ this.memory.get(key).add(new Rule(Rule.Operation.OR, bool));
+ else
+ this.memory.get(key).setValue(new Rule(Rule.Operation.OR, this.<Boolean>getValue(key) || bool));
+ }
+
+ @Override
+ public void set(final String key, final Object value) {
+ checkKeyValue(key, value);
+ if (this.inTask)
+ this.memory.get(key).add(new Rule(Rule.Operation.SET, value));
+ else
+ this.memory.get(key).setValue(new Rule(Rule.Operation.SET, value));
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.memoryString(this);
+ }
+
+ protected void setInTask(final boolean inTask) {
+ this.inTask = inTask;
+ }
+
+ protected void broadcastMemory(final JavaSparkContext sparkContext) {
+ this.broadcast.destroy(true); // do we need to block?
+ final Map<String, Object> toBroadcast = new HashMap<>();
+ this.memory.forEach((key, rule) -> {
+ if (null != rule.value().getObject())
+ toBroadcast.put(key, rule.value().getObject());
+ });
+ this.broadcast = sparkContext.broadcast(toBroadcast);
+ }
+
+ private void checkKeyValue(final String key, final Object value) {
+ if (!this.memoryKeys.contains(key))
+ throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
+ MemoryHelper.validateValue(value);
+ }
+
+ private <R> R getValue(final String key) {
+ return this.inTask ? (R) this.broadcast.value().get(key) : (R) this.memory.get(key).value().getObject();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMessenger.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMessenger.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMessenger.java
new file mode 100644
index 0000000..f32c684
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMessenger.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import scala.Tuple2;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkMessenger<M> implements Messenger<M> {
+
+ private Vertex vertex;
+ private Iterable<M> incomingMessages;
+ private List<Tuple2<Object, M>> outgoingMessages = new ArrayList<>();
+
+ public void setVertexAndIncomingMessages(final Vertex vertex, final Iterable<M> incomingMessages) {
+ this.vertex = vertex;
+ this.incomingMessages = incomingMessages;
+ this.outgoingMessages = new ArrayList<>();
+ }
+
+ public List<Tuple2<Object, M>> getOutgoingMessages() {
+ return this.outgoingMessages;
+ }
+
+ @Override
+ public Iterator<M> receiveMessages() {
+ return this.incomingMessages.iterator();
+ }
+
+ @Override
+ public void sendMessage(final MessageScope messageScope, final M message) {
+ if (messageScope instanceof MessageScope.Local) {
+ final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
+ final Traversal.Admin<Vertex, Edge> incidentTraversal = SparkMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.vertex);
+ final Direction direction = SparkMessenger.getOppositeDirection(incidentTraversal);
+ incidentTraversal.forEachRemaining(edge -> this.outgoingMessages.add(new Tuple2<>(edge.vertices(direction).next().id(), message)));
+ } else {
+ ((MessageScope.Global) messageScope).vertices().forEach(v -> this.outgoingMessages.add(new Tuple2<>(v.id(), message)));
+ }
+ }
+
+ ///////////
+
+ private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
+ incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
+ return (T) incidentTraversal;
+ }
+
+ private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
+ final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
+ return step.getDirection().opposite();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkReduceEmitter.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkReduceEmitter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkReduceEmitter.java
new file mode 100644
index 0000000..e2252fa
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkReduceEmitter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer;
+
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import scala.Tuple2;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class SparkReduceEmitter<OK, OV> implements MapReduce.ReduceEmitter<OK, OV> {
+
+ private List<Tuple2<OK, OV>> emissions = new ArrayList<>();
+
+ @Override
+ public void emit(final OK key, final OV value) {
+ this.emissions.add(new Tuple2<>(key, value));
+ }
+
+ public Iterator<Tuple2<OK, OV>> getEmissions() {
+ final Iterator<Tuple2<OK, OV>> iterator = this.emissions.iterator();
+ this.emissions = new ArrayList<>();
+ return iterator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputFormatRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputFormatRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputFormatRDD.java
new file mode 100644
index 0000000..adb080b
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputFormatRDD.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import scala.Tuple2;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class InputFormatRDD implements InputRDD {
+
+ @Override
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) {
+ final org.apache.hadoop.conf.Configuration hadoopConfiguration = ConfUtil.makeHadoopConfiguration(configuration);
+ return sparkContext.newAPIHadoopRDD(hadoopConfiguration,
+ (Class<InputFormat<NullWritable, VertexWritable>>) hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class),
+ NullWritable.class,
+ VertexWritable.class)
+ .mapToPair(tuple -> new Tuple2<>(tuple._2().get().id(), new VertexWritable(tuple._2().get())))
+ .reduceByKey((a, b) -> a); // if this is not done, then the graph is partitioned and you can have duplicate vertices
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/04f5651e/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDD.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDD.java
new file mode 100644
index 0000000..19d79a8
--- /dev/null
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/io/InputRDD.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.spark.process.computer.io;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+
+/**
+ * An InputRDD is used to read data from the underlying graph system and yield the respective adjacency list.
+ * Note that {@link InputFormatRDD} is a type of InputRDD that simply uses the specified {@link org.apache.hadoop.mapreduce.InputFormat} to generate the respective graphRDD.
+ *
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public interface InputRDD {
+
+ /**
+ * Read the graphRDD from the underlying graph system.
+ * @param configuration the configuration for the {@link org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.SparkGraphComputer}.
+ * @param sparkContext the Spark context with the requisite methods for generating a {@link JavaPairRDD}.
+ * @return an adjacency list representation of the underlying graph system.
+ */
+ public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext);
+}
[10/18] incubator-tinkerpop git commit: Spark, Giraph,
and Hadoop are now all split up and don't have such crazy pom.xml
dependency issues. Still need to find the minimum exclusions needed for Spark
and then will convert to Spark 1.4.1.
Posted by ok...@apache.org.
Spark, Giraph, and Hadoop are now all split up and don't have such crazy pom.xml dependency issues. Still need to find the minimum exclusions needed for Spark and then will convert to Spark 1.4.1.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/1303acfc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/1303acfc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/1303acfc
Branch: refs/heads/master
Commit: 1303acfcd56e588bfa7094b5b0cdcb81d71469f7
Parents: ac56b30
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 8 11:04:48 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 8 11:04:48 2015 -0600
----------------------------------------------------------------------
.../traverser/B_O_S_SE_SL_Traverser.java | 2 +-
spark-gremlin/pom.xml | 60 ++++++++++++++------
2 files changed, 43 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1303acfc/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/B_O_S_SE_SL_Traverser.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/B_O_S_SE_SL_Traverser.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/B_O_S_SE_SL_Traverser.java
index b69db85..5dc1313 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/B_O_S_SE_SL_Traverser.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/traverser/B_O_S_SE_SL_Traverser.java
@@ -121,6 +121,6 @@ public class B_O_S_SE_SL_Traverser<T> extends B_O_Traverser<T> {
&& ((B_O_S_SE_SL_Traverser) object).get().equals(this.t)
&& ((B_O_S_SE_SL_Traverser) object).getStepId().equals(this.getStepId())
&& ((B_O_S_SE_SL_Traverser) object).loops() == this.loops()
- && (null == this.sack || null != this.sideEffects.getSackMerger());
+ && (null == this.sack || (null != this.sideEffects && null != this.sideEffects.getSackMerger())); // hmmm... serialization in OLAP destroys the transient sideEffects
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/1303acfc/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index bf33ae5..f1c696e 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -46,28 +46,24 @@
<version>${project.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.giraph</groupId>
- <artifactId>giraph-core</artifactId>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
</exclusion>
<exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
</exclusion>
<exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -77,6 +73,27 @@
<artifactId>spark-core_2.10</artifactId>
<version>1.2.1</version>
<exclusions>
+ <!-- self conflicts -->
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
<!-- gremlin-core conflicts -->
<exclusion>
<groupId>org.slf4j</groupId>
@@ -95,17 +112,24 @@
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
+ <!-- hadoop conflicts -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
<!-- lgpl conflicts -->
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>findbugs</artifactId>
</exclusion>
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
</exclusions>
</dependency>
+ <!-- consistent dependencies -->
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.10.3</version>
+ </dependency>
<!-- TEST -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
[16/18] incubator-tinkerpop git commit: fixed self-conflict issues in
Apache Spark (again).
Posted by ok...@apache.org.
fixed self-conflict issues in Apache Spark (again).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/00963d16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/00963d16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/00963d16
Branch: refs/heads/master
Commit: 00963d16d9d7b1cc15e12a02999603a93e699e4b
Parents: d33a849 8986156
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Sep 9 10:33:04 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Sep 9 10:33:04 2015 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
CONTRIBUTING.asciidoc | 28 +-
RELEASE.asciidoc | 16 +-
docs/preprocessor/preprocess.sh | 8 +
docs/src/the-traversal.asciidoc | 4 +-
.../traversal/dsl/graph/GraphTraversal.java | 37 ++-
.../process/traversal/step/filter/DropStep.java | 5 +-
.../traversal/step/map/PropertyMapStep.java | 8 +
.../process/traversal/step/util/Parameters.java | 25 ++
.../strategy/decoration/PartitionStrategy.java | 280 ++++++++++++++++++-
.../tinkerpop/gremlin/structure/Element.java | 5 +
.../traversal/step/util/ParametersTest.java | 95 +++++++
.../PartitionStrategyTraverseTest.java | 72 +++--
.../gremlin/driver/AuthProperties.java | 34 ++-
.../tinkerpop/gremlin/driver/Cluster.java | 46 +++
.../tinkerpop/gremlin/driver/Handler.java | 2 +-
.../step/branch/GroovyRepeatTest.groovy | 2 +-
.../step/map/GroovyAddVertexTest.groovy | 20 +-
.../step/sideEffect/GroovyStoreTest.groovy | 2 +-
.../server/GremlinDriverIntegrateTest.java | 11 +-
.../traversal/step/map/AddVertexTest.java | 72 +++++
.../decoration/EventStrategyProcessTest.java | 4 +-
.../PartitionStrategyProcessTest.java | 217 ++++++++++++++
.../tinkerpop/gremlin/structure/EdgeTest.java | 38 ---
.../gremlin/structure/VertexPropertyTest.java | 38 ---
.../tinkerpop/gremlin/structure/VertexTest.java | 45 ---
.../structure/ExceptionCoverageTest.java | 5 +-
.../gremlin/neo4j/structure/Neo4jEdge.java | 14 +-
.../gremlin/neo4j/structure/Neo4jElement.java | 1 -
.../gremlin/neo4j/structure/Neo4jVertex.java | 6 -
.../neo4j/structure/Neo4jVertexProperty.java | 5 -
.../structure/trait/MultiMetaNeo4jTrait.java | 18 +-
spark-gremlin/pom.xml | 57 +++-
.../step/sideEffect/TinkerGraphStep.java | 7 +-
.../tinkergraph/structure/TinkerEdge.java | 6 +-
.../tinkergraph/structure/TinkerElement.java | 3 +
.../tinkergraph/structure/TinkerVertex.java | 10 +-
.../structure/TinkerVertexProperty.java | 2 +-
.../tinkergraph/structure/TinkerGraphTest.java | 37 +++
39 files changed, 1037 insertions(+), 249 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/00963d16/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --cc spark-gremlin/pom.xml
index af23904,0000000..d49f1d4
mode 100644,000000..100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@@ -1,306 -1,0 +1,337 @@@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkerpop</artifactId>
+ <version>3.1.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>spark-gremlin</artifactId>
+ <name>Apache TinkerPop :: Spark Gremlin</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-core</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>hadoop-gremlin</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- SPARK GRAPH COMPUTER -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>1.5.0</version>
+ <exclusions>
+ <!-- self conflicts -->
+ <exclusion>
++ <groupId>org.scala-lang</groupId>
++ <artifactId>scala-library</artifactId>
++ </exclusion>
++ <exclusion>
++ <groupId>org.scala-lang</groupId>
++ <artifactId>scala-reflect</artifactId>
++ </exclusion>
++ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- </exclusion>
- <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
++ <groupId>commons-lang</groupId>
++ <artifactId>commons-lang</artifactId>
++ </exclusion>
++ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
++ <exclusion>
++ <groupId>org.xerial.snappy</groupId>
++ <artifactId>snappy-java</artifactId>
++ </exclusion>
++ <exclusion>
++ <groupId>org.apache.curator</groupId>
++ <artifactId>curator-framework</artifactId>
++ </exclusion>
++ <exclusion>
++ <groupId>com.thoughtworks.paranamer</groupId>
++ <artifactId>paranamer</artifactId>
++ </exclusion>
+ <!-- gremlin-core conflicts -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ivy</groupId>
+ <artifactId>ivy</artifactId>
+ </exclusion>
+ <!-- gremlin-groovy conflicts -->
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <!-- hadoop conflicts -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ <!-- lgpl conflicts -->
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- consistent dependencies -->
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
- <version>2.10.3</version>
++ <version>2.10.5</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.4.4</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.4.4</version>
+ </dependency>
- <!-- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>2.4.4</version>
++ <dependency>
++ <groupId>commons-lang</groupId>
++ <artifactId>commons-lang</artifactId>
++ <version>2.6</version>
++ </dependency>
++ <dependency>
++ <groupId>org.xerial.snappy</groupId>
++ <artifactId>snappy-java</artifactId>
++ <version>1.1.1.7</version>
++ </dependency>
++ <!--<dependency>
++ <groupId>org.apache.curator</groupId>
++ <artifactId>curator-framework</artifactId>
++ <version>2.4.0</version>
+ </dependency>-->
++ <dependency>
++ <groupId>com.thoughtworks.paranamer</groupId>
++ <artifactId>paranamer</artifactId>
++ <version>2.6</version>
++ </dependency>
+ <!-- TEST -->
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-test</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy-test</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkergraph-gremlin</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <directory>${basedir}/target</directory>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <id>build-detached-assemblies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/assembly/standalone.xml</descriptor>
+ <descriptor>src/assembly/hadoop-job.xml</descriptor>
+ </descriptors>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.gmavenplus</groupId>
+ <artifactId>gmavenplus-plugin</artifactId>
+ <version>1.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>addSources</goal>
+ <goal>addTestSources</goal>
+ <goal>generateStubs</goal>
+ <goal>compile</goal>
+ <goal>testGenerateStubs</goal>
+ <goal>testCompile</goal>
+ <goal>removeStubs</goal>
+ <goal>removeTestStubs</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <invokeDynamic>true</invokeDynamic>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.5</version>
+ <configuration>
+ <archive>
+ <manifestEntries>
+ <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-core:1.2.1
+ </Gremlin-Plugin-Dependencies>
+ <!-- deletes the servlet-api jar from the path after install - causes conflicts -->
+ <Gremlin-Plugin-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
+ </Gremlin-Plugin-Paths>
+ <Gremlin-Lib-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
+ </Gremlin-Lib-Paths>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- log4j.configuration>log4j-core.properties</log4j.configuration> -->
+ <!--<argLine>-Xmx2048M</argLine>-->
+ <excludes>
+ <exclude>**/*IntegrateTest.java</exclude>
+ <exclude>**/*PerformanceTest.java</exclude>
+ <!-- this is technically a member of the integration test suite -->
+ <exclude>**/HadoopGremlinPluginTest.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
[09/18] incubator-tinkerpop git commit: pulled out giraph-gremlin
from hadoop-gremlin. HadoopGremlin builds nicely. Giraph too. Spark is still
causing problems.
Posted by ok...@apache.org.
pulled out giraph-gremlin from hadoop-gremlin. HadoopGremlin builds nicely. Giraph too. Spark is still causing problems.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ac56b309
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ac56b309
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ac56b309
Branch: refs/heads/master
Commit: ac56b3095c8d9ddb685fd1286e7d870db1739618
Parents: b2132b1
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Sep 8 10:49:19 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Sep 8 10:49:19 2015 -0600
----------------------------------------------------------------------
giraph-gremlin/pom.xml | 235 +++++++++++++++++++
.../giraph/process/computer/EmptyOutEdges.java | 80 +++++++
.../process/computer/GiraphComputeVertex.java | 50 ++++
.../process/computer/GiraphGraphComputer.java | 222 ++++++++++++++++++
.../giraph/process/computer/GiraphMemory.java | 218 +++++++++++++++++
.../process/computer/GiraphMessageCombiner.java | 62 +++++
.../process/computer/GiraphMessenger.java | 79 +++++++
.../process/computer/GiraphWorkerContext.java | 77 ++++++
.../process/computer/MemoryAggregator.java | 94 ++++++++
.../computer/io/GiraphVertexInputFormat.java | 65 +++++
.../computer/io/GiraphVertexOutputFormat.java | 65 +++++
.../process/computer/io/GiraphVertexReader.java | 67 ++++++
.../process/computer/io/GiraphVertexWriter.java | 57 +++++
.../giraph/process/HadoopGraphProvider.java | 161 +++++++++++++
...GiraphGraphComputerProcessIntegrateTest.java | 32 +++
.../computer/HadoopGiraphGraphProvider.java | 37 +++
...GraphComputerGroovyProcessIntegrateTest.java | 33 +++
.../GiraphHadoopGremlinPluginIntegrateTest.java | 32 +++
hadoop-gremlin/pom.xml | 50 ----
.../groovy/plugin/HadoopGremlinPlugin.java | 3 -
.../process/computer/giraph/EmptyOutEdges.java | 81 -------
.../computer/giraph/GiraphComputeVertex.java | 50 ----
.../computer/giraph/GiraphGraphComputer.java | 222 ------------------
.../process/computer/giraph/GiraphMemory.java | 218 -----------------
.../computer/giraph/GiraphMessageCombiner.java | 62 -----
.../computer/giraph/GiraphMessenger.java | 79 -------
.../computer/giraph/GiraphWorkerContext.java | 77 ------
.../computer/giraph/MemoryAggregator.java | 94 --------
.../giraph/io/GiraphVertexInputFormat.java | 65 -----
.../giraph/io/GiraphVertexOutputFormat.java | 65 -----
.../computer/giraph/io/GiraphVertexReader.java | 67 ------
.../computer/giraph/io/GiraphVertexWriter.java | 57 -----
.../gremlin/hadoop/structure/HadoopGraph.java | 48 ++--
.../gremlin/hadoop/HadoopGraphProvider.java | 22 +-
...GiraphGraphComputerProcessIntegrateTest.java | 32 ---
.../giraph/HadoopGiraphGraphProvider.java | 36 ---
...GraphComputerGroovyProcessIntegrateTest.java | 33 ---
.../GiraphHadoopGremlinPluginIntegrateTest.java | 33 ---
pom.xml | 1 +
spark-gremlin/pom.xml | 6 -
.../spark/process/HadoopGraphProvider.java | 1 -
41 files changed, 1701 insertions(+), 1367 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml
new file mode 100644
index 0000000..214e9c4
--- /dev/null
+++ b/giraph-gremlin/pom.xml
@@ -0,0 +1,235 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkerpop</artifactId>
+ <version>3.1.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>giraph-gremlin</artifactId>
+ <name>Apache TinkerPop :: Giraph Gremlin</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>hadoop-gremlin</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.giraph</groupId>
+ <artifactId>giraph-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- GIRAPH GRAPH COMPUTER -->
+ <dependency>
+ <groupId>org.apache.giraph</groupId>
+ <artifactId>giraph-core</artifactId>
+ <version>1.0.0</version>
+ <exclusions>
+ <!-- self conflicts -->
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <!-- conflicts with gremlin-core -->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <!-- gremlin-groovy conflicts -->
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ <!-- gremlin-test conflicts -->
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- consistent dependencies -->
+ <dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <version>3.1</version>
+ </dependency>
+ <!-- TEST -->
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-test</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>gremlin-groovy-test</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tinkerpop</groupId>
+ <artifactId>tinkergraph-gremlin</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <repositories>
+ <repository>
+ <id>hyracks-releases</id>
+ <url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public-releases/</url>
+ </repository>
+ </repositories>
+ <build>
+ <directory>${basedir}/target</directory>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <resources>
+ <resource>
+ <directory>${basedir}/src/main/resources</directory>
+ </resource>
+ </resources>
+ <testResources>
+ <testResource>
+ <directory>${basedir}/src/test/resources</directory>
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <executions>
+ <execution>
+ <id>build-detached-assemblies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <attach>false</attach>
+ <descriptors>
+ <descriptor>src/assembly/standalone.xml</descriptor>
+ <descriptor>src/assembly/hadoop-job.xml</descriptor>
+ </descriptors>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.gmavenplus</groupId>
+ <artifactId>gmavenplus-plugin</artifactId>
+ <version>1.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>addSources</goal>
+ <goal>addTestSources</goal>
+ <goal>generateStubs</goal>
+ <goal>compile</goal>
+ <goal>testGenerateStubs</goal>
+ <goal>testCompile</goal>
+ <goal>removeStubs</goal>
+ <goal>removeTestStubs</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <invokeDynamic>true</invokeDynamic>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.5</version>
+ <configuration>
+ <archive>
+ <manifestEntries>
+ <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-core:1.2.1
+ </Gremlin-Plugin-Dependencies>
+ <!-- deletes the servlet-api jar from the path after install - causes conflicts -->
+ <Gremlin-Plugin-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
+ </Gremlin-Plugin-Paths>
+ <Gremlin-Lib-Paths>servlet-api-2.5-6.1.14.jar=;servlet-api-2.5-20081211.jar=
+ </Gremlin-Lib-Paths>
+ </manifestEntries>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- log4j.configuration>log4j-core.properties</log4j.configuration> -->
+ <!--<argLine>-Xmx2048M</argLine>-->
+ <excludes>
+ <exclude>**/*IntegrateTest.java</exclude>
+ <exclude>**/*PerformanceTest.java</exclude>
+ <!-- this is technically a member of the integration test suite -->
+ <exclude>**/HadoopGremlinPluginTest.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java
new file mode 100644
index 0000000..cb649f6
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/EmptyOutEdges.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.giraph.edge.Edge;
+import org.apache.giraph.edge.OutEdges;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class EmptyOutEdges implements OutEdges<ObjectWritable, NullWritable> {
+
+ private static final EmptyOutEdges INSTANCE = new EmptyOutEdges();
+
+ public static EmptyOutEdges instance() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void initialize(final Iterable<Edge<ObjectWritable, NullWritable>> edges) {
+ }
+
+ @Override
+ public void initialize(final int capacity) {
+ }
+
+ @Override
+ public void initialize() {
+ }
+
+ @Override
+ public void add(final Edge<ObjectWritable, NullWritable> edge) {
+ }
+
+ @Override
+ public void remove(final ObjectWritable targetVertexId) {
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+
+ @Override
+ public Iterator<Edge<ObjectWritable, NullWritable>> iterator() {
+ return Collections.emptyIterator();
+ }
+
+ @Override
+ public void write(final DataOutput dataOutput) throws IOException {
+ }
+
+ @Override
+ public void readFields(final DataInput dataInput) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java
new file mode 100644
index 0000000..7f39c46
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphComputeVertex.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.giraph.graph.Vertex;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphComputeVertex extends Vertex<ObjectWritable, VertexWritable, NullWritable, ObjectWritable> {
+
+ public GiraphComputeVertex() {
+ }
+
+ public GiraphComputeVertex(final VertexWritable vertexWritable) {
+ final VertexWritable newWritable = new VertexWritable();
+ newWritable.set(vertexWritable.get());
+ this.initialize(new ObjectWritable<>(newWritable.get().id()), newWritable, EmptyOutEdges.instance());
+
+ }
+
+ @Override
+ public void compute(final Iterable<ObjectWritable> messages) {
+ final GiraphWorkerContext workerContext = (GiraphWorkerContext) this.getWorkerContext();
+ final VertexProgram<?> vertexProgram = workerContext.getVertexProgramPool().take();
+ vertexProgram.execute(ComputerGraph.vertexProgram(this.getValue().get(), vertexProgram), workerContext.getMessenger(this, messages.iterator()), workerContext.getMemory());
+ workerContext.getVertexProgramPool().offer(vertexProgram);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java
new file mode 100644
index 0000000..acd5628
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphGraphComputer.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.commons.configuration.BaseConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.FileConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.giraph.conf.GiraphConfiguration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.giraph.job.GiraphJob;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.tinkerpop.giraph.process.computer.io.GiraphVertexInputFormat;
+import org.apache.tinkerpop.giraph.process.computer.io.GiraphVertexOutputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MapReduceHelper;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.HadoopHelper;
+import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+
+import java.io.File;
+import java.io.NotSerializableException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
+import java.util.stream.Stream;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphGraphComputer extends AbstractHadoopGraphComputer implements GraphComputer, Tool {
+
+ protected GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
+ private MapMemory memory = new MapMemory();
+
+ public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
+ super(hadoopGraph);
+ final Configuration configuration = hadoopGraph.configuration();
+ configuration.getKeys().forEachRemaining(key -> this.giraphConfiguration.set(key, configuration.getProperty(key).toString()));
+ this.giraphConfiguration.setMasterComputeClass(GiraphMemory.class);
+ this.giraphConfiguration.setVertexClass(GiraphComputeVertex.class);
+ this.giraphConfiguration.setWorkerContextClass(GiraphWorkerContext.class);
+ this.giraphConfiguration.setOutEdgesClass(EmptyOutEdges.class);
+ this.giraphConfiguration.setClass(GiraphConstants.VERTEX_ID_CLASS.getKey(), ObjectWritable.class, ObjectWritable.class);
+ this.giraphConfiguration.setClass(GiraphConstants.VERTEX_VALUE_CLASS.getKey(), VertexWritable.class, VertexWritable.class);
+ this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
+ this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
+ this.giraphConfiguration.setVertexOutputFormatClass(GiraphVertexOutputFormat.class);
+ }
+
+ @Override
+ public GraphComputer program(final VertexProgram vertexProgram) {
+ super.program(vertexProgram);
+ this.memory.addVertexProgramMemoryComputeKeys(this.vertexProgram);
+ final BaseConfiguration apacheConfiguration = new BaseConfiguration();
+ vertexProgram.storeState(apacheConfiguration);
+ ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, this.giraphConfiguration);
+ this.vertexProgram.getMessageCombiner().ifPresent(combiner -> this.giraphConfiguration.setCombinerClass(GiraphMessageCombiner.class));
+ return this;
+ }
+
+ @Override
+ public Future<ComputerResult> submit() {
+ final long startTime = System.currentTimeMillis();
+ super.validateStatePriorToExecution();
+ return CompletableFuture.<ComputerResult>supplyAsync(() -> {
+ try {
+ final FileSystem fs = FileSystem.get(this.giraphConfiguration);
+ this.loadJars(fs);
+ fs.delete(new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION)), true);
+ ToolRunner.run(this, new String[]{});
+ } catch (final Exception e) {
+ //e.printStackTrace();
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+
+ this.memory.setRuntime(System.currentTimeMillis() - startTime);
+ return new DefaultComputerResult(HadoopHelper.getOutputGraph(this.hadoopGraph, this.resultGraph, this.persist), this.memory.asImmutable());
+ });
+ }
+
+ @Override
+ public int run(final String[] args) {
+ this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, this.persist.equals(Persist.EDGES));
+ try {
+ // it is possible to run graph computer without a vertex program (and thus, only map reduce jobs if they exist)
+ if (null != this.vertexProgram) {
+ // a way to verify in Giraph whether the traversal will go over the wire or not
+ try {
+ VertexProgram.createVertexProgram(this.hadoopGraph, ConfUtil.makeApacheConfiguration(this.giraphConfiguration));
+ } catch (IllegalStateException e) {
+ if (e.getCause() instanceof NumberFormatException)
+ throw new NotSerializableException("The provided traversal is not serializable and thus, can not be distributed across the cluster");
+ }
+ // prepare the giraph vertex-centric computing job
+ final GiraphJob job = new GiraphJob(this.giraphConfiguration, Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
+ // handle input paths (if any)
+ if (FileInputFormat.class.isAssignableFrom(this.giraphConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class))) {
+ final Path inputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
+ if (!FileSystem.get(this.giraphConfiguration).exists(inputPath)) // TODO: what about when the input is not a file input?
+ throw new IllegalArgumentException("The provided input path does not exist: " + inputPath);
+ FileInputFormat.setInputPaths(job.getInternalJob(), inputPath);
+ }
+ // handle output paths
+ final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
+ FileOutputFormat.setOutputPath(job.getInternalJob(), outputPath);
+ job.getInternalJob().setJarByClass(GiraphGraphComputer.class);
+ this.logger.info(Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
+ // execute the job and wait until it completes (if it fails, throw an exception)
+ if (!job.run(true))
+ throw new IllegalStateException("The GiraphGraphComputer job failed -- aborting all subsequent MapReduce jobs"); // how do I get the exception that occured?
+ // add vertex program memory values to the return memory
+ for (final String key : this.vertexProgram.getMemoryComputeKeys()) {
+ final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + key);
+ final ObjectWritableIterator iterator = new ObjectWritableIterator(this.giraphConfiguration, path);
+ if (iterator.hasNext()) {
+ this.memory.set(key, iterator.next().getValue());
+ }
+ FileSystem.get(this.giraphConfiguration).delete(path, true);
+ }
+ final Path path = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_ITERATION);
+ this.memory.setIteration((Integer) new ObjectWritableIterator(this.giraphConfiguration, path).next().getValue());
+ FileSystem.get(this.giraphConfiguration).delete(path, true);
+ }
+ // do map reduce jobs
+ this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT_HAS_EDGES, this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT_HAS_EDGES, true));
+ for (final MapReduce mapReduce : this.mapReducers) {
+ this.memory.addMapReduceMemoryKey(mapReduce);
+ MapReduceHelper.executeMapReduceJob(mapReduce, this.memory, this.giraphConfiguration);
+ }
+
+ // if no persistence, delete the map reduce output
+ if (this.persist.equals(Persist.NOTHING)) {
+ final Path outputPath = new Path(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION) + "/" + Constants.HIDDEN_G);
+ if (FileSystem.get(this.giraphConfiguration).exists(outputPath)) // TODO: what about when the output is not a file output?
+ FileSystem.get(this.giraphConfiguration).delete(outputPath, true);
+ }
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ return 0;
+ }
+
+ @Override
+ public void setConf(final org.apache.hadoop.conf.Configuration configuration) {
+ // TODO: is this necessary to implement?
+ }
+
+ @Override
+ public org.apache.hadoop.conf.Configuration getConf() {
+ return this.giraphConfiguration;
+ }
+
+ private void loadJars(final FileSystem fs) {
+ final String hadoopGremlinLibsRemote = "hadoop-gremlin-libs";
+ if (this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true)) {
+ final String hadoopGremlinLocalLibs = System.getenv(Constants.HADOOP_GREMLIN_LIBS);
+ if (null == hadoopGremlinLocalLibs)
+ this.logger.warn(Constants.HADOOP_GREMLIN_LIBS + " is not set -- proceeding regardless");
+ else {
+ final String[] paths = hadoopGremlinLocalLibs.split(":");
+ for (final String path : paths) {
+ final File file = new File(path);
+ if (file.exists()) {
+ Stream.of(file.listFiles()).filter(f -> f.getName().endsWith(Constants.DOT_JAR)).forEach(f -> {
+ try {
+ final Path jarFile = new Path(fs.getHomeDirectory() + "/" + hadoopGremlinLibsRemote + "/" + f.getName());
+ fs.copyFromLocalFile(new Path(f.getPath()), jarFile);
+ try {
+ DistributedCache.addArchiveToClassPath(jarFile, this.giraphConfiguration, fs);
+ } catch (final Exception e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ });
+ } else {
+ this.logger.warn(path + " does not reference a valid directory -- proceeding regardless");
+ }
+ }
+ }
+ }
+ }
+
+ public static void main(final String[] args) throws Exception {
+ final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
+ new GiraphGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java
new file mode 100644
index 0000000..5a56bd3
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMemory.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.master.MasterCompute;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphMemory extends MasterCompute implements Memory {
+
+ private VertexProgram<?> vertexProgram;
+ private GiraphWorkerContext worker;
+ private Set<String> memoryKeys;
+ private boolean isMasterCompute = true;
+ private long startTime = System.currentTimeMillis();
+
+ public GiraphMemory() {
+ // Giraph ReflectionUtils requires this to be public at minimum
+ }
+
+ public GiraphMemory(final GiraphWorkerContext worker, final VertexProgram<?> vertexProgram) {
+ this.worker = worker;
+ this.vertexProgram = vertexProgram;
+ this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
+ this.isMasterCompute = false;
+ }
+
+
+ @Override
+ public void initialize() {
+ // do not initialize aggregators here because the getConf() configuration is not available at this point
+ // use compute() initial iteration instead
+ }
+
+ @Override
+ public void compute() {
+ this.isMasterCompute = true;
+ if (0 == this.getSuperstep()) { // setup
+ final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getConf());
+ this.vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
+ this.memoryKeys = new HashSet<>(this.vertexProgram.getMemoryComputeKeys());
+ try {
+ for (final String key : this.memoryKeys) {
+ MemoryHelper.validateKey(key);
+ this.registerPersistentAggregator(key, MemoryAggregator.class);
+ }
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ this.vertexProgram.setup(this);
+ } else {
+ if (this.vertexProgram.terminate(this)) { // terminate
+ // write the memory to HDFS
+ final MapMemory memory = new MapMemory(this);
+ // a hack to get the last iteration memory values to stick
+ this.vertexProgram.terminate(memory);
+ final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
+ if (null != outputLocation) {
+ try {
+ for (final String key : this.keys()) {
+ final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + key), ObjectWritable.class, ObjectWritable.class);
+ writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.get(key)));
+ writer.close();
+ }
+ final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + Constants.HIDDEN_ITERATION), ObjectWritable.class, ObjectWritable.class);
+ writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.getIteration()));
+ writer.close();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ this.haltComputation();
+ }
+ }
+ }
+
+ @Override
+ public int getIteration() {
+ if (this.isMasterCompute) {
+ final int temp = (int) this.getSuperstep();
+ return temp == 0 ? temp : temp - 1;
+ } else {
+ return (int) this.worker.getSuperstep();
+ }
+ }
+
+ @Override
+ public long getRuntime() {
+ return System.currentTimeMillis() - this.startTime;
+ }
+
+ @Override
+ public Set<String> keys() {
+ return this.memoryKeys.stream().filter(this::exists).collect(Collectors.toSet());
+ }
+
+ @Override
+ public boolean exists(final String key) {
+ final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
+ return null != rule.getObject();
+ }
+
+ @Override
+ public <R> R get(final String key) throws IllegalArgumentException {
+ //this.checkKey(key);
+ final Rule rule = this.isMasterCompute ? this.getAggregatedValue(key) : this.worker.getAggregatedValue(key);
+ if (null == rule.getObject())
+ throw Memory.Exceptions.memoryDoesNotExist(key);
+ else
+ return rule.getObject();
+ }
+
+ @Override
+ public void set(final String key, Object value) {
+ this.checkKeyValue(key, value);
+ if (this.isMasterCompute)
+ this.setAggregatedValue(key, new Rule(Rule.Operation.SET, value));
+ else
+ this.worker.aggregate(key, new Rule(Rule.Operation.SET, value));
+ }
+
+ @Override
+ public void and(final String key, final boolean bool) {
+ this.checkKeyValue(key, bool);
+ if (this.isMasterCompute) { // only called on setup() and terminate()
+ Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
+ value = null == value ? bool : bool && value;
+ this.setAggregatedValue(key, new Rule(Rule.Operation.AND, value));
+ } else {
+ this.worker.aggregate(key, new Rule(Rule.Operation.AND, bool));
+ }
+ }
+
+ @Override
+ public void or(final String key, final boolean bool) {
+ this.checkKeyValue(key, bool);
+ if (this.isMasterCompute) { // only called on setup() and terminate()
+ Boolean value = this.<Rule>getAggregatedValue(key).<Boolean>getObject();
+ value = null == value ? bool : bool || value;
+ this.setAggregatedValue(key, new Rule(Rule.Operation.OR, value));
+ } else {
+ this.worker.aggregate(key, new Rule(Rule.Operation.OR, bool));
+ }
+ }
+
+ @Override
+ public void incr(final String key, final long delta) {
+ this.checkKeyValue(key, delta);
+ if (this.isMasterCompute) { // only called on setup() and terminate()
+ Number value = this.<Rule>getAggregatedValue(key).<Number>getObject();
+ value = null == value ? delta : value.longValue() + delta;
+ this.setAggregatedValue(key, new Rule(Rule.Operation.INCR, value));
+ } else {
+ this.worker.aggregate(key, new Rule(Rule.Operation.INCR, delta));
+ }
+ }
+
+ @Override
+ public void write(final DataOutput output) {
+ // no need to serialize the master compute as it gets its data from aggregators
+ // is this true?
+ }
+
+ @Override
+ public void readFields(final DataInput input) {
+ // no need to serialize the master compute as it gets its data from aggregators
+ // is this true?
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.memoryString(this);
+ }
+
+ private void checkKeyValue(final String key, final Object value) {
+ if (!this.memoryKeys.contains(key))
+ throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
+ MemoryHelper.validateValue(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java
new file mode 100644
index 0000000..8da835d
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessageCombiner.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.combiner.Combiner;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
+import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphMessageCombiner extends Combiner<ObjectWritable, ObjectWritable> implements ImmutableClassesGiraphConfigurable {
+
+ private MessageCombiner messageCombiner;
+ private ImmutableClassesGiraphConfiguration configuration;
+
+ @Override
+ public void combine(final ObjectWritable vertexIndex, final ObjectWritable originalMessage, final ObjectWritable messageToCombine) {
+ originalMessage.set(originalMessage.isEmpty() ?
+ messageToCombine.get() :
+ this.messageCombiner.combine(originalMessage.get(), messageToCombine.get()));
+ }
+
+ @Override
+ public ObjectWritable createInitialMessage() {
+ return ObjectWritable.empty();
+ }
+
+ @Override
+ public void setConf(final ImmutableClassesGiraphConfiguration configuration) {
+ this.configuration = configuration;
+ final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration);
+ this.messageCombiner = (MessageCombiner) VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().get();
+ }
+
+ @Override
+ public ImmutableClassesGiraphConfiguration getConf() {
+ return this.configuration;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java
new file mode 100644
index 0000000..ca3e100
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphMessenger.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StartStep;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphMessenger<M> implements Messenger<M> {
+
+ private GiraphComputeVertex giraphComputeVertex;
+ private Iterator<ObjectWritable<M>> messages;
+
+ public GiraphMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable<M>> messages) {
+ this.giraphComputeVertex = giraphComputeVertex;
+ this.messages = messages;
+ }
+
+ @Override
+ public Iterator<M> receiveMessages() {
+ return IteratorUtils.map(this.messages, ObjectWritable::get);
+ }
+
+ @Override
+ public void sendMessage(final MessageScope messageScope, final M message) {
+ if (messageScope instanceof MessageScope.Local) {
+ final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
+ final Traversal.Admin<Vertex, Edge> incidentTraversal = GiraphMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get(), this.giraphComputeVertex.getValue().get());
+ final Direction direction = GiraphMessenger.getOppositeDirection(incidentTraversal);
+ incidentTraversal.forEachRemaining(edge ->
+ this.giraphComputeVertex.sendMessage(
+ new ObjectWritable<>(edge.vertices(direction).next().id()),
+ new ObjectWritable<>(localMessageScope.getEdgeFunction().apply(message, edge))));
+ } else {
+ final MessageScope.Global globalMessageScope = (MessageScope.Global) messageScope;
+ globalMessageScope.vertices().forEach(vertex ->
+ this.giraphComputeVertex.sendMessage(new ObjectWritable<>(vertex.id()), new ObjectWritable<>(message)));
+ }
+ }
+
+ private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal<Vertex, Edge> incidentTraversal, final Vertex vertex) {
+ incidentTraversal.asAdmin().addStep(0, new StartStep<>(incidentTraversal.asAdmin(), vertex));
+ return (T) incidentTraversal;
+ }
+
+ private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
+ final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
+ return step.getDirection().opposite();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java
new file mode 100644
index 0000000..6ff7dc0
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/GiraphWorkerContext.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.giraph.worker.WorkerContext;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
+import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.ImmutableMemory;
+import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
+
+import java.util.Iterator;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphWorkerContext extends WorkerContext {
+
+ private VertexProgramPool vertexProgramPool;
+ private GiraphMemory memory;
+
+ public GiraphWorkerContext() {
+ // Giraph ReflectionUtils requires this to be public at minimum
+ }
+
+ public void preApplication() throws InstantiationException, IllegalAccessException {
+ final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration());
+ HadoopPools.initialize(apacheConfiguration);
+ final VertexProgram vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
+ this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
+ this.memory = new GiraphMemory(this, vertexProgram);
+ }
+
+ public void postApplication() {
+
+ }
+
+ public void preSuperstep() {
+ this.vertexProgramPool.workerIterationStart(new ImmutableMemory(this.memory));
+ }
+
+ public void postSuperstep() {
+ this.vertexProgramPool.workerIterationEnd(new ImmutableMemory(this.memory));
+ }
+
+ public VertexProgramPool getVertexProgramPool() {
+ return this.vertexProgramPool;
+ }
+
+ public GiraphMemory getMemory() {
+ return this.memory;
+ }
+
+ public GiraphMessenger getMessenger(final GiraphComputeVertex giraphComputeVertex, final Iterator<ObjectWritable> messages) {
+ return new GiraphMessenger(giraphComputeVertex, messages);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java
new file mode 100644
index 0000000..6526fab
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/MemoryAggregator.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer;
+
+import org.apache.giraph.aggregators.Aggregator;
+import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.Rule;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class MemoryAggregator implements Aggregator<Rule> {
+
+ private Object currentObject;
+ private Rule.Operation lastOperation = null;
+
+ public MemoryAggregator() {
+ this.currentObject = null;
+ }
+
+ @Override
+ public Rule getAggregatedValue() {
+ if (null == this.currentObject)
+ return createInitialValue();
+ else if (this.currentObject instanceof Long)
+ return new Rule(Rule.Operation.INCR, this.currentObject);
+ else
+ return new Rule(null == this.lastOperation ? Rule.Operation.NO_OP : this.lastOperation, this.currentObject);
+ }
+
+ @Override
+ public void setAggregatedValue(final Rule rule) {
+ this.currentObject = rule.getObject();
+ }
+
+ @Override
+ public void reset() {
+ this.currentObject = null;
+ }
+
+ @Override
+ public Rule createInitialValue() {
+ return new Rule(Rule.Operation.NO_OP, null);
+ }
+
+ @Override
+ public void aggregate(final Rule ruleWritable) {
+ final Rule.Operation rule = ruleWritable.getOperation();
+ final Object object = ruleWritable.getObject();
+ if (rule != Rule.Operation.NO_OP)
+ this.lastOperation = rule;
+
+ if (null == this.currentObject || rule.equals(Rule.Operation.SET)) {
+ this.currentObject = object;
+ } else {
+ if (rule.equals(Rule.Operation.INCR)) {
+ this.currentObject = (Long) this.currentObject + (Long) object;
+ } else if (rule.equals(Rule.Operation.AND)) {
+ this.currentObject = (Boolean) this.currentObject && (Boolean) object;
+ } else if (rule.equals(Rule.Operation.OR)) {
+ this.currentObject = (Boolean) this.currentObject || (Boolean) object;
+ } else if (rule.equals(Rule.Operation.NO_OP)) {
+ if (object instanceof Boolean) { // only happens when NO_OP booleans are being propagated will this occur
+ if (null == this.lastOperation) {
+ // do nothing ... why?
+ } else if (this.lastOperation.equals(Rule.Operation.AND)) {
+ this.currentObject = (Boolean) this.currentObject && (Boolean) object;
+ } else if (this.lastOperation.equals(Rule.Operation.OR)) {
+ this.currentObject = (Boolean) this.currentObject || (Boolean) object;
+ } else {
+ throw new IllegalStateException("This state should not have occurred: " + ruleWritable);
+ }
+ }
+ } else {
+ throw new IllegalArgumentException("The provided rule is unknown: " + ruleWritable);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java
new file mode 100644
index 0000000..2b3b723
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexInputFormat.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer.io;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.giraph.io.VertexInputFormat;
+import org.apache.giraph.io.VertexReader;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexInputFormat extends VertexInputFormat {
+
+ private InputFormat<NullWritable, VertexWritable> hadoopGraphInputFormat;
+
+ @Override
+ public List<InputSplit> getSplits(final JobContext context, final int minSplitCountHint) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ return this.hadoopGraphInputFormat.getSplits(context);
+ }
+
+ @Override
+ public VertexReader createVertexReader(final InputSplit split, final TaskAttemptContext context) throws IOException {
+ this.constructor(context.getConfiguration());
+ try {
+ return new GiraphVertexReader(this.hadoopGraphInputFormat.createRecordReader(split, context));
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private final void constructor(final Configuration configuration) {
+ if (null == this.hadoopGraphInputFormat) {
+ this.hadoopGraphInputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class, InputFormat.class), configuration);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java
new file mode 100644
index 0000000..0135bd5
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexOutputFormat.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer.io;
+
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+import org.apache.giraph.io.VertexOutputFormat;
+import org.apache.giraph.io.VertexWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexOutputFormat extends VertexOutputFormat {
+
+ private OutputFormat<NullWritable, VertexWritable> hadoopGraphOutputFormat;
+
+ @Override
+ public VertexWriter createVertexWriter(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ return new GiraphVertexWriter(this.hadoopGraphOutputFormat);
+ }
+
+ @Override
+ public void checkOutputSpecs(final JobContext context) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ this.hadoopGraphOutputFormat.checkOutputSpecs(context);
+ }
+
+ @Override
+ public OutputCommitter getOutputCommitter(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.constructor(context.getConfiguration());
+ return this.hadoopGraphOutputFormat.getOutputCommitter(context);
+ }
+
+ private final void constructor(final Configuration configuration) {
+ if (null == this.hadoopGraphOutputFormat) {
+ this.hadoopGraphOutputFormat = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, OutputFormat.class, OutputFormat.class), configuration);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java
new file mode 100644
index 0000000..7c8273c
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexReader.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer.io;
+
+import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.io.VertexReader;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.tinkerpop.giraph.process.computer.GiraphComputeVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexReader extends VertexReader {
+
+ private RecordReader<NullWritable, VertexWritable> recordReader;
+
+ public GiraphVertexReader(final RecordReader<NullWritable, VertexWritable> recordReader) {
+ this.recordReader = recordReader;
+ }
+
+ @Override
+ public void initialize(final InputSplit inputSplit, final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.recordReader.initialize(inputSplit, context);
+ }
+
+ @Override
+ public boolean nextVertex() throws IOException, InterruptedException {
+ return this.recordReader.nextKeyValue();
+ }
+
+ @Override
+ public Vertex getCurrentVertex() throws IOException, InterruptedException {
+ return new GiraphComputeVertex(this.recordReader.getCurrentValue());
+ }
+
+ @Override
+ public void close() throws IOException {
+ this.recordReader.close();
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ return this.recordReader.getProgress();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java
new file mode 100644
index 0000000..812386d
--- /dev/null
+++ b/giraph-gremlin/src/main/java/org/apache/tinkerpop/giraph/process/computer/io/GiraphVertexWriter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.giraph.process.computer.io;
+
+import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.io.VertexWriter;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.tinkerpop.giraph.process.computer.GiraphComputeVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
+
+import java.io.IOException;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class GiraphVertexWriter extends VertexWriter {
+ private final OutputFormat<NullWritable, VertexWritable> outputFormat;
+ private RecordWriter<NullWritable, VertexWritable> recordWriter;
+
+ public GiraphVertexWriter(final OutputFormat<NullWritable, VertexWritable> outputFormat) {
+ this.outputFormat = outputFormat;
+ }
+
+ @Override
+ public void initialize(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.recordWriter = this.outputFormat.getRecordWriter(context);
+ }
+
+ @Override
+ public void close(final TaskAttemptContext context) throws IOException, InterruptedException {
+ this.recordWriter.close(context);
+ }
+
+ @Override
+ public void writeVertex(final Vertex vertex) throws IOException, InterruptedException {
+ this.recordWriter.write(NullWritable.get(), ((GiraphComputeVertex) vertex).getValue());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
new file mode 100644
index 0000000..ae0ae10
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/HadoopGraphProvider.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tinkerpop.gremlin.giraph.process;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.giraph.conf.GiraphConstants;
+import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.TestHelper;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopEdge;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopElement;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertex;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopVertexProperty;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
+import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class HadoopGraphProvider extends AbstractGraphProvider {
+
+ private static final Random RANDOM = new Random();
+ private boolean graphSONInput = false;
+
+ public static Map<String, String> PATHS = new HashMap<>();
+ private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{
+ add(HadoopEdge.class);
+ add(HadoopElement.class);
+ add(HadoopGraph.class);
+ add(HadoopProperty.class);
+ add(HadoopVertex.class);
+ add(HadoopVertexProperty.class);
+ }};
+
+ static {
+ try {
+ final List<String> kryoResources = Arrays.asList(
+ "tinkerpop-modern.kryo",
+ "grateful-dead.kryo",
+ "tinkerpop-classic.kryo",
+ "tinkerpop-crew.kryo");
+ for (final String fileName : kryoResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> graphsonResources = Arrays.asList(
+ "tinkerpop-modern.json",
+ "grateful-dead.json",
+ "tinkerpop-classic.json",
+ "tinkerpop-crew.json");
+ for (final String fileName : graphsonResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+
+ final List<String> scriptResources = Arrays.asList(
+ "tinkerpop-classic.txt",
+ "script-input.groovy",
+ "script-output.groovy",
+ "grateful-dead.txt",
+ "script-input-grateful-dead.groovy",
+ "script-output-grateful-dead.groovy");
+ for (final String fileName : scriptResources) {
+ PATHS.put(fileName, TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, "").getAbsolutePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
+ this.graphSONInput = RANDOM.nextBoolean();
+ return new HashMap<String, Object>() {{
+ put(Graph.GRAPH, HadoopGraph.class.getName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, graphSONInput ? GraphSONInputFormat.class.getCanonicalName() : GryoInputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+ put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
+ put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+ /// giraph configuration
+ put(GiraphConstants.MIN_WORKERS, 1);
+ put(GiraphConstants.MAX_WORKERS, 1);
+ put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
+ put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
+ put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
+ put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
+ put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
+ put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
+ put("mapred.reduce.tasks", 4);
+ put("giraph.vertexOutputFormatThreadSafe", false);
+ put("giraph.numOutputThreads", 3);
+ }};
+ }
+
+ @Override
+ public void clear(final Graph graph, final Configuration configuration) throws Exception {
+ if (graph != null)
+ graph.close();
+ }
+
+ @Override
+ public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) {
+ if (loadGraphWith != null) this.loadGraphDataViaHadoopConfig(graph, loadGraphWith.value());
+ }
+
+ @Override
+ public Set<Class> getImplementations() {
+ return IMPLEMENTATION;
+ }
+
+ public void loadGraphDataViaHadoopConfig(final Graph g, final LoadGraphWith.GraphData graphData) {
+ final String type = this.graphSONInput ? "json" : "kryo";
+
+ if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("grateful-dead." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-modern." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-classic." + type));
+ } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) {
+ ((HadoopGraph) g).configuration().setInputLocation(PATHS.get("tinkerpop-crew." + type));
+ } else {
+ throw new RuntimeException("Could not load graph with " + graphData);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java
new file mode 100644
index 0000000..857f3b8
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(ProcessComputerSuite.class)
+@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
+public class GiraphGraphComputerProcessIntegrateTest {
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
new file mode 100644
index 0000000..8b124e5
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/HadoopGiraphGraphProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer;
+
+import org.apache.tinkerpop.giraph.process.computer.GiraphGraphComputer;
+import org.apache.tinkerpop.gremlin.GraphProvider;
+import org.apache.tinkerpop.gremlin.giraph.process.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@GraphProvider.Descriptor(computer = GiraphGraphComputer.class)
+public final class HadoopGiraphGraphProvider extends HadoopGraphProvider {
+
+ public GraphTraversalSource traversal(final Graph graph) {
+ return GraphTraversalSource.build().engine(ComputerTraversalEngine.build().computer(GiraphGraphComputer.class)).create(graph);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ac56b309/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java
new file mode 100644
index 0000000..9b01185
--- /dev/null
+++ b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.giraph.process.computer.groovy;
+
+import org.apache.tinkerpop.gremlin.GraphProviderClass;
+import org.apache.tinkerpop.gremlin.giraph.process.computer.HadoopGiraphGraphProvider;
+import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.process.GroovyProcessComputerSuite;
+import org.junit.runner.RunWith;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+@RunWith(GroovyProcessComputerSuite.class)
+@GraphProviderClass(provider = HadoopGiraphGraphProvider.class, graph = HadoopGraph.class)
+public class GiraphGraphComputerGroovyProcessIntegrateTest {
+}
[15/18] incubator-tinkerpop git commit: bumped to Spark 1.5.0 and
fixed a couple random issues in the test suite.
Posted by ok...@apache.org.
bumped to Spark 1.5.0 and fixed a couple random issues in the test suite.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/d33a849a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/d33a849a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/d33a849a
Branch: refs/heads/master
Commit: d33a849aa6fc3deddb22ef899d5132b1425b5b58
Parents: 65d5a61
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Sep 9 10:01:35 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Sep 9 10:01:35 2015 -0600
----------------------------------------------------------------------
.../gremlin/process/traversal/step/filter/GroovyHasTest.groovy | 4 ++--
.../gremlin/process/traversal/step/map/GroovyAddEdgeTest.groovy | 4 ++--
.../process/traversal/step/map/GroovyAddVertexTest.groovy | 2 +-
.../tinkerpop/gremlin/process/traversal/step/filter/HasTest.java | 3 +++
spark-gremlin/pom.xml | 2 +-
5 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d33a849a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyHasTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyHasTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyHasTest.groovy
index 73e6b79..f5a5054 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyHasTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyHasTest.groovy
@@ -76,13 +76,13 @@ public abstract class GroovyHasTest {
@Override
public Traversal<Vertex, Vertex> get_g_VX1X_out_hasIdX2X(final Object v1Id, final Object v2Id) {
- TraversalScriptHelper.compute(" g.V(v1Id).out.hasId(v2Id)", g, "v1Id", v1Id, "v2Id", v2Id);
+ TraversalScriptHelper.compute("g.V(v1Id).out.hasId(v2Id)", g, "v1Id", v1Id, "v2Id", v2Id);
}
@Override
public Traversal<Vertex, Vertex> get_g_VX1X_out_hasIdX2_3X(
final Object v1Id, final Object v2Id, final Object v3Id) {
- TraversalScriptHelper.compute(" g.V(v1Id).out.hasId(v2Id, v3Id)", g, "v1Id", v1Id, "v2Id", v2Id, "v3Id", v3Id);
+ TraversalScriptHelper.compute("g.V(v1Id).out.hasId(v2Id, v3Id)", g, "v1Id", v1Id, "v2Id", v2Id, "v3Id", v3Id);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d33a849a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddEdgeTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddEdgeTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddEdgeTest.groovy
index 15c5286..176e731 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddEdgeTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddEdgeTest.groovy
@@ -32,13 +32,13 @@ public abstract class GroovyAddEdgeTest {
@Override
public Traversal<Vertex, Edge> get_g_VX1X_asXaX_outXcreatedX_addEXcreatedByX_toXaX(final Object v1Id) {
- TraversalScriptHelper.compute("g.V(${v1Id}).as('a').out('created').addE('createdBy').to('a')", g, "v1Id", v1Id)
+ TraversalScriptHelper.compute("g.V(v1Id).as('a').out('created').addE('createdBy').to('a')", g, "v1Id", v1Id)
}
@Override
public Traversal<Vertex, Edge> get_g_VX1X_asXaX_outXcreatedX_addEXcreatedByX_toXaX_propertyXweight_2X(
final Object v1Id) {
- TraversalScriptHelper.compute("g.V(${v1Id}).as('a').out('created').addE('createdBy').to('a').property('weight', 2)", g, "v1Id", v1Id)
+ TraversalScriptHelper.compute("g.V(v1Id).as('a').out('created').addE('createdBy').to('a').property('weight', 2)", g, "v1Id", v1Id)
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d33a849a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddVertexTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddVertexTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddVertexTest.groovy
index b42f4ce..785739a 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddVertexTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/map/GroovyAddVertexTest.groovy
@@ -33,7 +33,7 @@ public abstract class GroovyAddVertexTest {
@Override
public Traversal<Vertex, Vertex> get_g_VX1X_addVXanimalX_propertyXage_selectXaX_byXageXX_propertyXname_puppyX(
final Object v1Id) {
- TraversalScriptHelper.compute("g.V(${v1Id}).as('a').addV('animal').property('age', select('a').by('age')).property('name', 'puppy')", g, "v1Id", v1Id);
+ TraversalScriptHelper.compute("g.V(v1Id).as('a').addV('animal').property('age', select('a').by('age')).property('name', 'puppy')", g, "v1Id", v1Id);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d33a849a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/HasTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/HasTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/HasTest.java
index 29c00f7..ed4a5e8 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/HasTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/HasTest.java
@@ -22,8 +22,10 @@ import org.apache.tinkerpop.gremlin.FeatureRequirement;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
+import org.apache.tinkerpop.gremlin.process.IgnoreEngine;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Element;
@@ -156,6 +158,7 @@ public abstract class HasTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
+ @IgnoreEngine(TraversalEngine.Type.COMPUTER)
public void g_VX1X_hasXage_gt_30X() {
Arrays.asList(get_g_VX1X_hasXage_gt_30X(convertToVertexId("marko")), get_g_VXv1X_hasXage_gt_30X(convertToVertexId("marko"))).forEach(traversal -> {
printTraversalForm(traversal);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d33a849a/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index ac6c532..af23904 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -92,7 +92,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
- <version>1.4.1</version>
+ <version>1.5.0</version>
<exclusions>
<!-- self conflicts -->
<exclusion>
[06/18] incubator-tinkerpop git commit: more work trying to get
hadoop_split dependencies working. this is a nest to say the least.
Posted by ok...@apache.org.
more work trying to get hadoop_split dependencies working. this is a nest to say the least.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/b2132b1f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/b2132b1f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/b2132b1f
Branch: refs/heads/master
Commit: b2132b1fae9bae418db06fff3037f44a8e8b8574
Parents: 718dd63
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Sep 3 14:57:11 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Sep 3 14:57:11 2015 -0600
----------------------------------------------------------------------
spark-gremlin/pom.xml | 65 ++++++++------------
.../spark/process/HadoopGraphProvider.java | 11 ----
2 files changed, 26 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2132b1f/spark-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 277ce11..645c97f 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -44,34 +44,39 @@
<groupId>org.apache.tinkerpop</groupId>
<artifactId>hadoop-gremlin</artifactId>
<version>${project.version}</version>
- </dependency>
- <!-- SPARK GRAPH COMPUTER -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
- <version>1.2.1</version>
<exclusions>
- <!-- self conflicts -->
<exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
+ <groupId>org.apache.giraph</groupId>
+ <artifactId>giraph-core</artifactId>
</exclusion>
<exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
</exclusion>
+ </exclusions>
+ </dependency>
+ <!-- SPARK GRAPH COMPUTER -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.10</artifactId>
+ <version>1.2.1</version>
+ <exclusions>
<!-- gremlin-core conflicts -->
<exclusion>
<groupId>org.slf4j</groupId>
@@ -90,35 +95,17 @@
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
- <!-- hadoop conflicts -->
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- <!-- giraph conflicts -->
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
<!-- lgpl conflicts -->
<exclusion>
<groupId>com.google.code.findbugs</groupId>
<artifactId>findbugs</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </exclusion>
</exclusions>
</dependency>
- <!-- consistent dependencies -->
-<!-- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>2.10.3</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.5.13.Final</version>
- </dependency>
--->
<!-- TEST -->
<dependency>
<groupId>org.apache.tinkerpop</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/b2132b1f/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
index f085678..333636e 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/HadoopGraphProvider.java
@@ -20,7 +20,6 @@
package org.apache.tinkerpop.gremlin.spark.process;
import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.TestHelper;
@@ -110,16 +109,6 @@ public class HadoopGraphProvider extends AbstractGraphProvider {
put(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "hadoop-gremlin/target/test-output");
put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
- /// giraph configuration
- put(GiraphConstants.MIN_WORKERS, 1);
- put(GiraphConstants.MAX_WORKERS, 1);
- put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
- put(GiraphConstants.ZOOKEEPER_SERVER_PORT.getKey(), 2181); // you must have a local zookeeper running on this port
- put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 3);
- put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 3);
- put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
put("mapred.reduce.tasks", 4);
//put("giraph.vertexOutputFormatThreadSafe", false);
//put("giraph.numOutputThreads", 3);