You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2018/03/23 21:08:47 UTC
[1/2] tinkerpop git commit: TINKERPOP-1930 Remove Giraph
Repository: tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1930 [created] ed5924c33
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
deleted file mode 100644
index 56428f3..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMemory.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.master.MasterCompute;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.MemoryHelper;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-import org.javatuples.Pair;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.BinaryOperator;
-import java.util.stream.Collectors;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMemory extends MasterCompute implements Memory {
-
- private VertexProgram<?> vertexProgram;
- private GiraphWorkerContext worker;
- private Map<String, MemoryComputeKey> memoryComputeKeys;
- private boolean inExecute = false;
- private long startTime = System.currentTimeMillis();
-
- public GiraphMemory() {
- // Giraph ReflectionUtils requires this to be public at minimum
- }
-
- public GiraphMemory(final GiraphWorkerContext worker, final VertexProgram<?> vertexProgram) {
- this.worker = worker;
- this.vertexProgram = vertexProgram;
- this.memoryComputeKeys = new HashMap<>();
- this.vertexProgram.getMemoryComputeKeys().forEach(key -> this.memoryComputeKeys.put(key.getKey(), key));
- this.inExecute = true;
- }
-
-
- @Override
- public void initialize() {
- // do not initialize aggregators here because the getConf() configuration is not available at this point
- // use compute() initial iteration instead
- }
-
- @Override
- public void compute() {
- this.inExecute = false;
- if (0 == this.getSuperstep()) { // setup
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getConf());
- this.vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- this.memoryComputeKeys = new HashMap<>();
- this.vertexProgram.getMemoryComputeKeys().forEach(key -> this.memoryComputeKeys.put(key.getKey(), key));
- try {
- for (final MemoryComputeKey key : this.memoryComputeKeys.values()) {
- this.registerPersistentAggregator(key.getKey(), MemoryAggregator.class);
- }
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- this.vertexProgram.setup(this);
- } else {
- // a hack to get the last iteration memory values to stick
- final PassThroughMemory memory = new PassThroughMemory(this);
- if (this.vertexProgram.terminate(memory)) { // terminate
- final String outputLocation = this.getConf().get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
- if (null != outputLocation) {
- try {
- for (final String key : this.keys()) {
- if (!this.memoryComputeKeys.get(key).isTransient()) { // do not write transient memory keys to disk
- final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + key), ObjectWritable.class, ObjectWritable.class);
- writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.get(key)));
- writer.close();
- }
- }
- // written for GiraphGraphComputer to read and then is deleted by GiraphGraphComputer
- final SequenceFile.Writer writer = SequenceFile.createWriter(FileSystem.get(this.getConf()), this.getConf(), new Path(outputLocation + "/" + Constants.HIDDEN_ITERATION), ObjectWritable.class, ObjectWritable.class);
- writer.append(ObjectWritable.getNullObjectWritable(), new ObjectWritable<>(memory.getIteration()));
- writer.close();
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
- this.haltComputation();
- }
- }
- }
-
- @Override
- public int getIteration() {
- if (this.inExecute) {
- return (int) this.worker.getSuperstep();
- } else {
- final int temp = (int) this.getSuperstep();
- return temp == 0 ? temp : temp - 1;
- }
- }
-
- @Override
- public long getRuntime() {
- return System.currentTimeMillis() - this.startTime;
- }
-
- @Override
- public Set<String> keys() {
- return this.memoryComputeKeys.values().stream().filter(key -> this.exists(key.getKey())).map(MemoryComputeKey::getKey).collect(Collectors.toSet());
- }
-
- @Override
- public boolean exists(final String key) {
- if (this.inExecute && this.memoryComputeKeys.containsKey(key) && !this.memoryComputeKeys.get(key).isBroadcast())
- return false;
- final ObjectWritable value = this.inExecute ? this.worker.getAggregatedValue(key) : this.getAggregatedValue(key);
- return null != value && !value.isEmpty();
- }
-
- @Override
- public <R> R get(final String key) throws IllegalArgumentException {
- if (!this.memoryComputeKeys.containsKey(key))
- throw Memory.Exceptions.memoryDoesNotExist(key);
- if (this.inExecute && !this.memoryComputeKeys.get(key).isBroadcast())
- throw Memory.Exceptions.memoryDoesNotExist(key);
- final ObjectWritable<Pair<BinaryOperator, Object>> value = this.inExecute ?
- this.worker.<ObjectWritable<Pair<BinaryOperator, Object>>>getAggregatedValue(key) :
- this.<ObjectWritable<Pair<BinaryOperator, Object>>>getAggregatedValue(key);
- if (null == value || value.isEmpty())
- throw Memory.Exceptions.memoryDoesNotExist(key);
- else
- return (R) value.get().getValue1();
- }
-
- @Override
- public void set(final String key, final Object value) {
- this.checkKeyValue(key, value);
- if (this.inExecute)
- throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
- this.setAggregatedValue(key, new ObjectWritable<>(new Pair<>(this.memoryComputeKeys.get(key).getReducer(), value)));
- }
-
- @Override
- public void add(final String key, final Object value) {
- this.checkKeyValue(key, value);
- if (!this.inExecute)
- throw Memory.Exceptions.memoryAddOnlyDuringVertexProgramExecute(key);
- this.worker.aggregate(key, new ObjectWritable<>(new Pair<>(this.memoryComputeKeys.get(key).getReducer(), value)));
- }
-
- @Override
- public void write(final DataOutput output) {
- // all aggregator data is propagated through writables
- }
-
- @Override
- public void readFields(final DataInput input) {
- // all aggregator data is propagated through writables
- }
-
- @Override
- public String toString() {
- return StringFactory.memoryString(this);
- }
-
- private void checkKeyValue(final String key, final Object value) {
- if (!this.memoryComputeKeys.containsKey(key))
- throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
- MemoryHelper.validateValue(value);
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java
deleted file mode 100644
index 4d725b2..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessageCombiner.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfigurable;
-import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.MessageCombiner;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMessageCombiner extends org.apache.giraph.combiner.MessageCombiner<ObjectWritable, ObjectWritable> implements ImmutableClassesGiraphConfigurable {
-
- private MessageCombiner messageCombiner;
- private ImmutableClassesGiraphConfiguration configuration;
-
- @Override
- public void combine(final ObjectWritable vertexIndex, final ObjectWritable originalMessage, final ObjectWritable messageToCombine) {
- originalMessage.set(originalMessage.isEmpty() ?
- messageToCombine.get() :
- this.messageCombiner.combine(originalMessage.get(), messageToCombine.get()));
- }
-
- @Override
- public ObjectWritable createInitialMessage() {
- return ObjectWritable.empty();
- }
-
- @Override
- public void setConf(final ImmutableClassesGiraphConfiguration configuration) {
- this.configuration = configuration;
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(configuration);
- this.messageCombiner = (MessageCombiner) VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration).getMessageCombiner().get();
- }
-
- @Override
- public ImmutableClassesGiraphConfiguration getConf() {
- return this.configuration;
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java
deleted file mode 100644
index 36e641e..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphMessenger.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
-import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexStep;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphMessenger<M> implements Messenger<M> {
-
- private GiraphVertex giraphVertex;
- private GiraphComputation giraphComputation;
- private Iterator<ObjectWritable<M>> messages;
-
- public GiraphMessenger(final GiraphVertex giraphVertex, final GiraphComputation giraphComputation, final Iterator<ObjectWritable<M>> messages) {
- this.giraphVertex = giraphVertex;
- this.giraphComputation = giraphComputation;
- this.messages = messages;
- }
-
- @Override
- public Iterator<M> receiveMessages() {
- return IteratorUtils.map(this.messages, ObjectWritable::get);
- }
-
- @Override
- public void sendMessage(final MessageScope messageScope, final M message) {
- if (messageScope instanceof MessageScope.Local) {
- final MessageScope.Local<M> localMessageScope = (MessageScope.Local) messageScope;
- final Traversal.Admin<Vertex, Edge> incidentTraversal = GiraphMessenger.setVertexStart(localMessageScope.getIncidentTraversal().get().asAdmin(), this.giraphVertex.getValue().get());
- final Direction direction = GiraphMessenger.getOppositeDirection(incidentTraversal);
-
- // handle processing for BOTH given TINKERPOP-1862 where the target of the message is the one opposite
- // the current vertex
- incidentTraversal.forEachRemaining(edge -> {
- if (direction.equals(Direction.IN) || direction.equals(Direction.OUT))
- this.giraphComputation.sendMessage(
- new ObjectWritable<>(edge.vertices(direction).next().id()),
- new ObjectWritable<>(localMessageScope.getEdgeFunction().apply(message, edge)));
- else
- this.giraphComputation.sendMessage(
- new ObjectWritable<>(edge instanceof StarGraph.StarOutEdge ? edge.inVertex().id() : edge.outVertex().id()),
- new ObjectWritable<>(localMessageScope.getEdgeFunction().apply(message, edge)));
- });
- } else {
- final MessageScope.Global globalMessageScope = (MessageScope.Global) messageScope;
- globalMessageScope.vertices().forEach(vertex ->
- this.giraphComputation.sendMessage(new ObjectWritable<>(vertex.id()), new ObjectWritable<>(message)));
- }
- }
-
- private static <T extends Traversal.Admin<Vertex, Edge>> T setVertexStart(final Traversal.Admin<Vertex, Edge> incidentTraversal, final Vertex vertex) {
- incidentTraversal.asAdmin().addStart(incidentTraversal.getTraverserGenerator().generate(vertex, incidentTraversal.getStartStep(), 1l));
- return (T) incidentTraversal;
- }
-
- private static Direction getOppositeDirection(final Traversal.Admin<Vertex, Edge> incidentTraversal) {
- final VertexStep step = TraversalHelper.getLastStepOfAssignableClass(VertexStep.class, incidentTraversal).get();
- return step.getDirection().opposite();
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphVertex.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphVertex.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphVertex.java
deleted file mode 100644
index 3a95fae..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphVertex.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.giraph.graph.DefaultVertex;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertex extends DefaultVertex<ObjectWritable, VertexWritable, NullWritable> {
-
- public GiraphVertex() {
- }
-
- public GiraphVertex(final VertexWritable vertexWritable) {
- final VertexWritable newWritable = new VertexWritable();
- newWritable.set(vertexWritable.get());
- this.initialize(new ObjectWritable<>(newWritable.get().id()), newWritable, EmptyOutEdges.instance());
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
deleted file mode 100644
index 0122ab4..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphWorkerContext.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.commons.configuration.Configuration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.worker.WorkerContext;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.ImmutableMemory;
-import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
-
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphWorkerContext extends WorkerContext {
-
- private VertexProgramPool vertexProgramPool;
- private GiraphMemory memory;
-
- public GiraphWorkerContext() {
- // Giraph ReflectionUtils requires this to be public at minimum
- }
-
- public void preApplication() throws InstantiationException, IllegalAccessException {
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration());
- KryoShimServiceLoader.applyConfiguration(apacheConfiguration);
- final VertexProgram vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
- this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
- this.memory = new GiraphMemory(this, vertexProgram);
- }
-
- public void postApplication() {
-
- }
-
- public void preSuperstep() {
- this.vertexProgramPool.workerIterationStart(new ImmutableMemory(this.memory));
- }
-
- public void postSuperstep() {
- this.vertexProgramPool.workerIterationEnd(new ImmutableMemory(this.memory));
- }
-
- public VertexProgramPool getVertexProgramPool() {
- return this.vertexProgramPool;
- }
-
- public GiraphMemory getMemory() {
- return this.memory;
- }
-
- public GiraphMessenger getMessenger(final GiraphVertex giraphVertex, final GiraphComputation giraphComputation, final Iterator<ObjectWritable> messages) {
- return new GiraphMessenger(giraphVertex, giraphComputation, messages);
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java
deleted file mode 100644
index 4929546..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/MemoryAggregator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.giraph.aggregators.Aggregator;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.javatuples.Pair;
-
-import java.util.function.BinaryOperator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class MemoryAggregator implements Aggregator<ObjectWritable<Pair<BinaryOperator, Object>>> {
-
- private ObjectWritable<Pair<BinaryOperator, Object>> currentObject = ObjectWritable.<Pair<BinaryOperator, Object>>empty();
-
- public MemoryAggregator() { // for Giraph serialization
-
- }
-
- @Override
- public ObjectWritable<Pair<BinaryOperator, Object>> getAggregatedValue() {
- return this.currentObject;
- }
-
- @Override
- public void setAggregatedValue(final ObjectWritable<Pair<BinaryOperator, Object>> object) {
- if (null != object)
- this.currentObject = object;
- }
-
- @Override
- public void aggregate(final ObjectWritable<Pair<BinaryOperator, Object>> object) {
- if (null == object)
- return;
- else if (this.currentObject.isEmpty())
- this.currentObject = object;
- else if (!object.isEmpty())
- this.currentObject.set(new Pair<>(object.get().getValue0(), object.get().getValue0().apply(this.currentObject.get().getValue1(), object.get().getValue1())));
- }
-
- @Override
- public void reset() {
- this.currentObject = ObjectWritable.<Pair<BinaryOperator, Object>>empty();
- }
-
- @Override
- public ObjectWritable<Pair<BinaryOperator, Object>> createInitialValue() {
- return ObjectWritable.<Pair<BinaryOperator, Object>>empty();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassThroughMemory.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassThroughMemory.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassThroughMemory.java
deleted file mode 100644
index dc40e7b..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/PassThroughMemory.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.tinkerpop.gremlin.process.computer.Memory;
-import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class PassThroughMemory implements Memory.Admin {
-
- private final GiraphMemory giraphMemory;
- private long runtime = 0l;
- private int iteration = -1;
- private final Map<String, Object> memoryMap = new HashMap<>();
-
- public PassThroughMemory(final GiraphMemory giraphMemory) {
- this.giraphMemory = giraphMemory;
- giraphMemory.keys().forEach(key -> this.memoryMap.put(key, giraphMemory.get(key)));
- this.iteration = giraphMemory.getIteration();
- }
-
- @Override
- public Set<String> keys() {
- return this.memoryMap.keySet();
- }
-
- @Override
- public <R> R get(final String key) throws IllegalArgumentException {
- final R r = (R) this.memoryMap.get(key);
- if (null == r)
- throw Memory.Exceptions.memoryDoesNotExist(key);
- else
- return r;
- }
-
- @Override
- public void set(final String key, Object value) {
- this.memoryMap.put(key, value);
- this.giraphMemory.set(key, value);
- }
-
- @Override
- public int getIteration() {
- return this.iteration;
- }
-
- @Override
- public long getRuntime() {
- return this.runtime;
- }
-
- @Override
- public void add(final String key, final Object value) {
- this.giraphMemory.add(key, value);
- }
-
- @Override
- public String toString() {
- return StringFactory.memoryString(this);
- }
-
- @Override
- public void incrIteration() {
- this.iteration = this.iteration + 1;
- }
-
- @Override
- public void setIteration(final int iteration) {
- this.iteration = iteration;
- }
-
- @Override
- public void setRuntime(long runtime) {
- this.runtime = runtime;
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexInputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexInputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexInputFormat.java
deleted file mode 100644
index 5900663..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexInputFormat.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.structure.io;
-
-import org.apache.giraph.io.VertexInputFormat;
-import org.apache.giraph.io.VertexReader;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.GraphFilterInputFormat;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexInputFormat extends VertexInputFormat {
-
- @Override
- public void checkInputSpecs(final Configuration configuration) {
-
- }
-
- @Override
- public List<InputSplit> getSplits(final JobContext context, final int minSplitCountHint) throws IOException, InterruptedException {
- return new GraphFilterInputFormat().getSplits(context);
- }
-
- @Override
- public VertexReader createVertexReader(final InputSplit split, final TaskAttemptContext context) throws IOException {
- try {
- final GiraphVertexReader reader = new GiraphVertexReader();
- reader.initialize(split, context);
- return reader;
- } catch (final InterruptedException e) {
- throw new IOException(e.getMessage(), e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexOutputFormat.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexOutputFormat.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexOutputFormat.java
deleted file mode 100644
index 9881c77..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexOutputFormat.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.structure.io;
-
-import org.apache.giraph.io.VertexOutputFormat;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.OutputCommitter;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexOutputFormat extends VertexOutputFormat {
-
- @Override
- public VertexWriter createVertexWriter(final TaskAttemptContext context) throws IOException, InterruptedException {
- return new GiraphVertexWriter();
- }
-
- @Override
- public void checkOutputSpecs(final JobContext context) throws IOException, InterruptedException {
- final Configuration configuration = context.getConfiguration();
- ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, OutputFormat.class, OutputFormat.class), configuration).checkOutputSpecs(context);
- }
-
- @Override
- public OutputCommitter getOutputCommitter(final TaskAttemptContext context) throws IOException, InterruptedException {
- final Configuration configuration = context.getConfiguration();
- return ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, OutputFormat.class, OutputFormat.class), configuration).getOutputCommitter(context);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexReader.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexReader.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexReader.java
deleted file mode 100644
index 5335980..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexReader.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.structure.io;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.io.VertexReader;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphVertex;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.GraphFilterRecordReader;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexReader extends VertexReader {
-
- private RecordReader<NullWritable, VertexWritable> recordReader;
-
- public GiraphVertexReader() {
- this.recordReader = new GraphFilterRecordReader();
- }
-
- @Override
- public void initialize(final InputSplit inputSplit, final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordReader.initialize(inputSplit, context);
- }
-
- @Override
- public boolean nextVertex() throws IOException, InterruptedException {
- return this.recordReader.nextKeyValue();
- }
-
- @Override
- public Vertex getCurrentVertex() throws IOException, InterruptedException {
- return new GiraphVertex(this.recordReader.getCurrentValue());
- }
-
- @Override
- public void close() throws IOException {
- this.recordReader.close();
- }
-
- @Override
- public float getProgress() throws IOException, InterruptedException {
- return this.recordReader.getProgress();
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexWriter.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexWriter.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexWriter.java
deleted file mode 100644
index 3c94137..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphVertexWriter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.structure.io;
-
-import org.apache.giraph.graph.Vertex;
-import org.apache.giraph.io.VertexWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphVertex;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramHelper;
-import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph;
-
-import java.io.IOException;
-import java.util.stream.Collectors;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphVertexWriter extends VertexWriter {
- private RecordWriter<NullWritable, VertexWritable> recordWriter;
- private String[] transientComputeKeys;
-
- public GiraphVertexWriter() {
-
- }
-
- @Override
- public void initialize(final TaskAttemptContext context) throws IOException, InterruptedException {
- final Configuration configuration = context.getConfiguration();
- this.recordWriter = ReflectionUtils.newInstance(configuration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, OutputFormat.class, OutputFormat.class), configuration).getRecordWriter(context);
- this.transientComputeKeys = VertexProgramHelper.vertexComputeKeysAsArray(((VertexProgram<?>) VertexProgram.createVertexProgram(EmptyGraph.instance(), ConfUtil.makeApacheConfiguration(configuration))).getVertexComputeKeys().stream().
- filter(VertexComputeKey::isTransient).
- collect(Collectors.toSet()));
- }
-
- @Override
- public void close(final TaskAttemptContext context) throws IOException, InterruptedException {
- this.recordWriter.close(context);
- }
-
- @Override
- public void writeVertex(final Vertex vertex) throws IOException, InterruptedException {
- ((GiraphVertex) vertex).getValue().get().dropVertexProperties(this.transientComputeKeys); // remove all transient compute keys before writing to OutputFormat
- this.recordWriter.write(NullWritable.get(), ((GiraphVertex) vertex).getValue());
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin b/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin
deleted file mode 100644
index 9a92905..0000000
--- a/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin
+++ /dev/null
@@ -1 +0,0 @@
-org.apache.tinkerpop.gremlin.giraph.jsr223.GiraphGremlinPlugin
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinIntegrateTest.java
deleted file mode 100644
index 955649c..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinIntegrateTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.giraph;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphHadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(GiraphGremlinSuite.class)
-@GraphProviderClass(provider = GiraphHadoopGraphProvider.class, graph = HadoopGraph.class)
-public class GiraphGremlinIntegrateTest {
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinSuite.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinSuite.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinSuite.java
deleted file mode 100644
index 2fe0e4f..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphGremlinSuite.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.giraph;
-
-import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
-import org.apache.tinkerpop.gremlin.giraph.structure.io.GiraphIoRegistryCheck;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
-import org.junit.runners.model.InitializationError;
-import org.junit.runners.model.RunnerBuilder;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphGremlinSuite extends AbstractGremlinSuite {
- public GiraphGremlinSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
- super(klass, builder, new Class<?>[]{GiraphIoRegistryCheck.class}, new Class<?>[]{GiraphIoRegistryCheck.class}, true, TraversalEngine.Type.COMPUTER);
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphHadoopGremlinIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphHadoopGremlinIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphHadoopGremlinIntegrateTest.java
deleted file mode 100644
index ba9e12d..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/GiraphHadoopGremlinIntegrateTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphHadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGremlinSuite;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(HadoopGremlinSuite.class)
-@GraphProviderClass(provider = GiraphHadoopGraphProvider.class, graph = HadoopGraph.class)
-public class GiraphHadoopGremlinIntegrateTest {
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java
deleted file mode 100644
index b6da750..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputerProcessIntegrateTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.tinkerpop.gremlin.GraphProviderClass;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite;
-import org.junit.runner.RunWith;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@RunWith(ProcessComputerSuite.class)
-@GraphProviderClass(provider = GiraphHadoopGraphProvider.class, graph = HadoopGraph.class)
-public class GiraphGraphComputerProcessIntegrateTest {
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphHadoopGraphProvider.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphHadoopGraphProvider.java
deleted file mode 100644
index 8eae4f1..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphHadoopGraphProvider.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.tinkerpop.gremlin.GraphProvider;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
-import org.apache.tinkerpop.gremlin.structure.Graph;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-@GraphProvider.Descriptor(computer = GiraphGraphComputer.class)
-public final class GiraphHadoopGraphProvider extends HadoopGraphProvider {
-
- @Override
- public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
- final Map<String, Object> config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith);
- config.put("mapreduce.job.reduces", 2);
- /// giraph configuration
- config.put(GiraphConstants.LOCAL_TEST_MODE.getKey(), true); // local testing can only spawn one worker
- config.put(GiraphConstants.MIN_WORKERS, 1);
- config.put(GiraphConstants.MAX_WORKERS, 1);
- config.put(GiraphConstants.SPLIT_MASTER_WORKER.getKey(), false);
- config.put(GiraphConstants.ZOOKEEPER_IS_EXTERNAL.getKey(), false);
- config.put(GiraphConstants.NETTY_SERVER_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- config.put(GiraphConstants.NETTY_CLIENT_USE_EXECUTION_HANDLER.getKey(), false); // this prevents so many integration tests running out of threads
- config.put(GiraphConstants.NETTY_USE_DIRECT_MEMORY.getKey(), true);
- config.put(GiraphConstants.NUM_INPUT_THREADS.getKey(), 2);
- config.put(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 2);
- config.put(GiraphConstants.MAX_MASTER_SUPERSTEP_WAIT_MSECS.getKey(), TimeUnit.MINUTES.toMillis(60L));
- config.put(GiraphConstants.VERTEX_OUTPUT_FORMAT_THREAD_SAFE.getKey(), false);
- config.put(GiraphConstants.NUM_OUTPUT_THREADS.getKey(), 1);
- return config;
- }
-
- @Override
- public GraphTraversalSource traversal(final Graph graph) {
- return graph.traversal().withComputer(GiraphGraphComputer.class);
- }
-
- @Override
- public GraphComputer getGraphComputer(final Graph graph) {
- return graph.compute(GiraphGraphComputer.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphIoRegistryCheck.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphIoRegistryCheck.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphIoRegistryCheck.java
deleted file mode 100644
index 0a9dc81..0000000
--- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/structure/io/GiraphIoRegistryCheck.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.giraph.structure.io;
-
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.AbstractIoRegistryCheck;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public class GiraphIoRegistryCheck extends AbstractIoRegistryCheck {
-
- @Before
- public void setup() throws Exception {
- super.setup();
- KryoShimServiceLoader.close();
- HadoopPools.close();
- }
-
- @After
- public void tearDown() throws Exception {
- super.tearDown();
- KryoShimServiceLoader.close();
- HadoopPools.close();
- }
-
- @Test
- public void shouldSupportGryoV1d0IoRegistry() throws Exception {
- super.checkGryoV1d0IoRegistryCompliance((HadoopGraph) graph, GiraphGraphComputer.class);
- }
-
- @Test
- public void shouldSupportGryoV3d0IoRegistry() throws Exception {
- super.checkGryoV3d0IoRegistryCompliance((HadoopGraph) graph, GiraphGraphComputer.class);
- }
-
- @Test
- public void shouldSupportGraphSONIoRegistry() throws Exception {
- super.checkGraphSONIoRegistryCompliance((HadoopGraph) graph, GiraphGraphComputer.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/resources/giraph-site.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/resources/giraph-site.xml b/giraph-gremlin/src/test/resources/giraph-site.xml
deleted file mode 100644
index 9862296..0000000
--- a/giraph-gremlin/src/test/resources/giraph-site.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
- <property>
- <name>giraph.zkServerlistPollMsecs</name>
- <value>500</value>
- </property>
- <property>
- <name>giraph.logLevel</name>
- <value>info</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/resources/log4j-silent.properties
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/resources/log4j-silent.properties b/giraph-gremlin/src/test/resources/log4j-silent.properties
deleted file mode 100644
index 1825bb0..0000000
--- a/giraph-gremlin/src/test/resources/log4j-silent.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# this file should always have logging set to OFF. it seems, however, that an appender of some sort is
-# required or else some logs throw error and use other log4j.properties files on the path.
-log4j.rootLogger=OFF, stdout
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/test/resources/log4j-test.properties b/giraph-gremlin/src/test/resources/log4j-test.properties
deleted file mode 100644
index a3e679c..0000000
--- a/giraph-gremlin/src/test/resources/log4j-test.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-log4j.rootLogger=WARN, stdout
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=[%p] %C - %m%n
-
-log4j.logger.org.apache.hadoop=ERROR
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/GremlinPlugin.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/GremlinPlugin.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/GremlinPlugin.java
index 99e6bd9..8f0ce9d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/GremlinPlugin.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/jsr223/GremlinPlugin.java
@@ -30,8 +30,8 @@ public interface GremlinPlugin {
/**
* The name of the module. This name should be unique (use a namespaced approach) as naming clashes will
* prevent proper module operations. Modules developed by TinkerPop will be prefixed with "tinkerpop."
- * For example, TinkerPop's implementation of Giraph would be named "tinkerpop.giraph". If Facebook were
- * to do their own implementation the implementation might be called "facebook.giraph".
+ * For example, TinkerPop's implementation of Spark would be named "tinkerpop.spark". If Facebook were
+ * to do their own implementation the implementation might be called "facebook.spark".
*/
public String getName();
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/hadoop-gremlin/conf/hadoop-graphson.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-graphson.properties b/hadoop-gremlin/conf/hadoop-graphson.properties
index c37cf28..c2f660b 100644
--- a/hadoop-gremlin/conf/hadoop-graphson.properties
+++ b/hadoop-gremlin/conf/hadoop-graphson.properties
@@ -33,12 +33,6 @@ spark.master=local[4]
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator
-#####################################
-# GiraphGraphComputer Configuration #
-#####################################
-giraph.minWorkers=2
-giraph.maxWorkers=2
-
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-grateful-gryo.properties b/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
index 92ed942..f28ce9d 100644
--- a/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
+++ b/hadoop-gremlin/conf/hadoop-grateful-gryo.properties
@@ -30,15 +30,3 @@ spark.executor.memory=1g
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator
-#####################################
-# GiraphGraphComputer Configuration #
-#####################################
-giraph.minWorkers=1
-giraph.maxWorkers=1
-giraph.useOutOfCoreGraph=true
-giraph.useOutOfCoreMessages=true
-mapred.map.child.java.opts=-Xmx1024m
-mapred.reduce.child.java.opts=-Xmx1024m
-giraph.numInputThreads=4
-giraph.numComputeThreads=4
-giraph.maxMessagesInMemory=100000
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/hadoop-gremlin/conf/hadoop-gryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-gryo.properties b/hadoop-gremlin/conf/hadoop-gryo.properties
index c156a98..31286d8 100644
--- a/hadoop-gremlin/conf/hadoop-gryo.properties
+++ b/hadoop-gremlin/conf/hadoop-gryo.properties
@@ -42,21 +42,5 @@ spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.Gryo
# spark.eventLog.dir=/tmp/spark-event-logs
# spark.ui.killEnabled=true
-#####################################
-# GiraphGraphComputer Configuration #
-#####################################
-giraph.minWorkers=2
-giraph.maxWorkers=2
-giraph.useOutOfCoreGraph=true
-giraph.useOutOfCoreMessages=true
-mapreduce.map.java.opts=-Xmx1024m
-mapreduce.reduce.java.opts=-Xmx1024m
-giraph.numInputThreads=2
-giraph.numComputeThreads=2
-# giraph.maxPartitionsInMemory=1
-# giraph.userPartitionCount=2
-## MapReduce of GiraphGraphComputer ##
-# mapreduce.job.maps=2
-# mapreduce.job.reduces=1
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/hadoop-gremlin/conf/hadoop-script.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-script.properties b/hadoop-gremlin/conf/hadoop-script.properties
index 88bf6d9..24d847b 100644
--- a/hadoop-gremlin/conf/hadoop-script.properties
+++ b/hadoop-gremlin/conf/hadoop-script.properties
@@ -38,21 +38,5 @@ spark.kryo.registrator=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.Gryo
# spark.eventLog.dir=/tmp/spark-event-logs
# spark.ui.killEnabled=true
-#####################################
-# GiraphGraphComputer Configuration #
-#####################################
-giraph.minWorkers=2
-giraph.maxWorkers=2
-giraph.useOutOfCoreGraph=true
-giraph.useOutOfCoreMessages=true
-mapreduce.map.java.opts=-Xmx1024m
-mapreduce.reduce.java.opts=-Xmx1024m
-giraph.numInputThreads=2
-giraph.numComputeThreads=2
-# giraph.maxPartitionsInMemory=1
-# giraph.userPartitionCount=2
-## MapReduce of GiraphGraphComputer ##
-# mapreduce.job.maps=2
-# mapreduce.job.reduces=1
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
index 2fc5a66..53477b8 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/Constants.java
@@ -45,7 +45,6 @@ public final class Constants {
public static final String GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE = "gremlin.hadoop.jarsInDistributedCache";
public static final String HIDDEN_G = Graph.Hidden.hide("g");
public static final String GREMLIN_HADOOP_JOB_PREFIX = "HadoopGremlin: ";
- public static final String GREMLIN_HADOOP_GIRAPH_JOB_PREFIX = "HadoopGremlin(Giraph): ";
// public static final String GREMLIN_HADOOP_MAP_REDUCE_JOB_PREFIX = "HadoopGremlin(MapReduce): ";
public static final String GREMLIN_HADOOP_SPARK_JOB_PREFIX = "HadoopGremlin(Spark): ";
public static final String HADOOP_GREMLIN_LIBS = "HADOOP_GREMLIN_LIBS";
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index 72eee73..ea8334b 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -69,21 +69,6 @@ import java.util.stream.Stream;
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
computers = {"ALL"})
@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
- method = "g_V_matchXa_knows_b__c_knows_bX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
- method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
-@Graph.OptOut(
- test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$Traversals",
- method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX",
- reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.",
- computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
-@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals",
method = "g_V_both_both_count",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.",
@@ -144,7 +129,7 @@ import java.util.stream.Stream;
test = "org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionComputerTest",
method = "*",
reason = "This test makes use of a sideEffect to enforce when a thread interruption is triggered and thus isn't applicable to HadoopGraph",
- computers = {"org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer", "org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"})
+ computers = {"org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer"})
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$CountMatchTraversals",
method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count",
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a1abade..ef2be48 100644
--- a/pom.xml
+++ b/pom.xml
@@ -123,7 +123,6 @@ limitations under the License.
<module>gremlin-dotnet</module>
<module>hadoop-gremlin</module>
<module>spark-gremlin</module>
- <module>giraph-gremlin</module>
<module>neo4j-gremlin</module>
<module>gremlin-driver</module>
<module>gremlin-console</module>
[2/2] tinkerpop git commit: TINKERPOP-1930 Remove Giraph
Posted by sp...@apache.org.
TINKERPOP-1930 Remove Giraph
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ed5924c3
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ed5924c3
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ed5924c3
Branch: refs/heads/TINKERPOP-1930
Commit: ed5924c3373946d9ccef3f30a370f8f28dcea52d
Parents: 23ca117
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Mar 23 08:20:05 2018 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Mar 23 08:40:35 2018 -0400
----------------------------------------------------------------------
docs/preprocessor/install-plugins.sh | 2 +-
docs/preprocessor/preprocess-file.sh | 7 +-
docs/site/home/gremlin.html | 3 +-
docs/site/home/index.html | 1 -
docs/site/home/providers.html | 12 +-
docs/src/dev/provider/index.asciidoc | 4 +-
.../src/reference/gremlin-applications.asciidoc | 9 -
.../reference/implementations-giraph.asciidoc | 145 ---------
.../implementations-hadoop-end.asciidoc | 144 +--------
.../implementations-hadoop-start.asciidoc | 42 +--
docs/src/reference/index.asciidoc | 1 -
.../tutorials/getting-started/index.asciidoc | 14 +-
docs/src/upgrade/release-3.4.x.asciidoc | 8 +
giraph-gremlin/pom.xml | 258 ----------------
giraph-gremlin/src/assembly/hadoop-job.xml | 41 ---
giraph-gremlin/src/assembly/standalone.xml | 50 ---
.../giraph/jsr223/GiraphGremlinPlugin.java | 58 ----
.../giraph/process/computer/EmptyOutEdges.java | 80 -----
.../process/computer/GiraphComputation.java | 47 ---
.../process/computer/GiraphGraphComputer.java | 303 -------------------
.../giraph/process/computer/GiraphMemory.java | 198 ------------
.../process/computer/GiraphMessageCombiner.java | 61 ----
.../process/computer/GiraphMessenger.java | 90 ------
.../giraph/process/computer/GiraphVertex.java | 39 ---
.../process/computer/GiraphWorkerContext.java | 78 -----
.../process/computer/MemoryAggregator.java | 68 -----
.../process/computer/PassThroughMemory.java | 99 ------
.../structure/io/GiraphVertexInputFormat.java | 58 ----
.../structure/io/GiraphVertexOutputFormat.java | 55 ----
.../giraph/structure/io/GiraphVertexReader.java | 68 -----
.../giraph/structure/io/GiraphVertexWriter.java | 71 -----
...pache.tinkerpop.gremlin.jsr223.GremlinPlugin | 1 -
.../giraph/GiraphGremlinIntegrateTest.java | 33 --
.../gremlin/giraph/GiraphGremlinSuite.java | 35 ---
.../GiraphHadoopGremlinIntegrateTest.java | 33 --
...GiraphGraphComputerProcessIntegrateTest.java | 32 --
.../computer/GiraphHadoopGraphProvider.java | 68 -----
.../structure/io/GiraphIoRegistryCheck.java | 64 ----
.../src/test/resources/giraph-site.xml | 12 -
.../src/test/resources/log4j-silent.properties | 23 --
.../src/test/resources/log4j-test.properties | 23 --
.../tinkerpop/gremlin/jsr223/GremlinPlugin.java | 4 +-
hadoop-gremlin/conf/hadoop-graphson.properties | 6 -
.../conf/hadoop-grateful-gryo.properties | 12 -
hadoop-gremlin/conf/hadoop-gryo.properties | 16 -
hadoop-gremlin/conf/hadoop-script.properties | 16 -
.../tinkerpop/gremlin/hadoop/Constants.java | 1 -
.../gremlin/hadoop/structure/HadoopGraph.java | 17 +-
pom.xml | 1 -
49 files changed, 38 insertions(+), 2473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/preprocessor/install-plugins.sh
----------------------------------------------------------------------
diff --git a/docs/preprocessor/install-plugins.sh b/docs/preprocessor/install-plugins.sh
index 990dbe4..0a7ca31 100755
--- a/docs/preprocessor/install-plugins.sh
+++ b/docs/preprocessor/install-plugins.sh
@@ -25,7 +25,7 @@ TMP_DIR=$3
INSTALL_TEMPLATE="docs/preprocessor/install-plugins.groovy"
INSTALL_FILE="${TMP_DIR}/install-plugins.groovy"
-plugins=("hadoop-gremlin" "spark-gremlin" "giraph-gremlin" "neo4j-gremlin")
+plugins=("hadoop-gremlin" "spark-gremlin" "neo4j-gremlin")
# plugins=()
pluginsCount=${#plugins[@]}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/preprocessor/preprocess-file.sh
----------------------------------------------------------------------
diff --git a/docs/preprocessor/preprocess-file.sh b/docs/preprocessor/preprocess-file.sh
index d5076f1..ca1f205 100755
--- a/docs/preprocessor/preprocess-file.sh
+++ b/docs/preprocessor/preprocess-file.sh
@@ -110,7 +110,7 @@ if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then
mv ext/spark-gremlin .ext/
cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'SparkGremlinPlugin' > .ext/plugins.txt
;;
- "implementations-hadoop-start" | "implementations-hadoop-end" | "implementations-spark" | "implementations-giraph" | "olap-spark-yarn")
+ "implementations-hadoop-start" | "implementations-hadoop-end" | "implementations-spark" | "olap-spark-yarn")
# deactivate Neo4j plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars
mkdir .ext
mv ext/neo4j-gremlin .ext/
@@ -121,9 +121,8 @@ if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then
mkdir .ext
mv ext/neo4j-gremlin .ext/
mv ext/spark-gremlin .ext/
- mv ext/giraph-gremlin .ext/
mv ext/hadoop-gremlin .ext/
- cat ext/plugins.txt | tee .ext/plugins.all | grep -v 'Neo4jGremlinPlugin\|SparkGremlinPlugin\|GiraphGremlinPlugin\|HadoopGremlinPlugin' > .ext/plugins.txt
+ cat ext/plugins.txt | tee .ext/plugins.all | grep -v 'Neo4jGremlinPlugin\|SparkGremlinPlugin\|HadoopGremlinPlugin' > .ext/plugins.txt
;;
esac
@@ -135,7 +134,7 @@ if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then
awk -f ${AWK_SCRIPTS}/prepare.awk |
awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" -v PYTHONPATH="${TP_HOME}/gremlin-python/target/classes/Lib" |
awk -f ${AWK_SCRIPTS}/progressbar.awk -v tpl=${AWK_SCRIPTS}/progressbar.groovy.template |
- HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/giraph-gremlin/lib:${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh |
+ HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh |
${lb} awk -f ${AWK_SCRIPTS}/ignore.awk |
${lb} awk -f ${AWK_SCRIPTS}/prettify.awk |
${lb} awk -f ${AWK_SCRIPTS}/cleanup.awk |
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/site/home/gremlin.html
----------------------------------------------------------------------
diff --git a/docs/site/home/gremlin.html b/docs/site/home/gremlin.html
index bedddf3..6524650 100644
--- a/docs/site/home/gremlin.html
+++ b/docs/site/home/gremlin.html
@@ -339,8 +339,7 @@ g.V().hasLabel("person").
GraphTraversalSource g;
g = graph.traversal(); // local OLTP
g = graph.traversal().withRemote(DriverRemoteConnection.using("server.yaml")) // remote OLTP
-g = graph.traversal().withComputer(SparkGraphComputer.class); // distributed OLAP
-g = graph.traversal().withComputer(GiraphGraphComputer.class); // distributed OLAP</code>
+g = graph.traversal().withComputer(SparkGraphComputer.class); // distributed OLAP/code>
</pre>
</div>
<br/>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/site/home/index.html
----------------------------------------------------------------------
diff --git a/docs/site/home/index.html b/docs/site/home/index.html
index 5b76cc3..e45b227 100644
--- a/docs/site/home/index.html
+++ b/docs/site/home/index.html
@@ -222,7 +222,6 @@ limitations under the License.
<li><a href="https://github.com/MartinHaeusler/chronos/tree/master/org.chronos.chronograph">ChronoGraph</a> - A versioned graph database.</li>
<li><a href="http://www.datastax.com/products/datastax-enterprise-graph">DSEGraph</a> - DataStax graph database with OLTP and OLAP support.</li>
<li><a href="https://grakn.ai/">GRAKN.AI</a> - Distributed OLTP/OLAP knowledge graph system.</li>
- <li><a href="http://tinkerpop.apache.org/docs/current/reference/#giraphgraphcomputer">Hadoop (Giraph)</a> - OLAP graph processor using Giraph.</li>
<li><a href="http://tinkerpop.apache.org/docs/current/reference/#sparkgraphcomputer">Hadoop (Spark)</a> - OLAP graph processor using Spark.</li>
<li><a href="https://github.com/rayokota/hgraphdb">HGraphDB</a> - OLTP graph database running on Apache HBase.</li>
<li><a href="https://console.ng.bluemix.net/catalog/services/ibm-graph/">IBM Graph</a> - OLTP graph database as a service.</li>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/site/home/providers.html
----------------------------------------------------------------------
diff --git a/docs/site/home/providers.html b/docs/site/home/providers.html
index 269b07d..d0bc59a 100644
--- a/docs/site/home/providers.html
+++ b/docs/site/home/providers.html
@@ -75,7 +75,7 @@ limitations under the License.
<br/>
<li><strong>The GraphComputer (optional)</strong>: All OLAP-based graph processors must implement the primary graph interfaces mentioned above as well as a set of parallel-processing
message-passing interfaces. However, it is possible for a data system to only implement the primary graph interfaces and still provide OLAP support to their users by integrating their
- system with an existing <code>GraphComputer</code> implementation such as, for example, <code>SparkGraphComputer</code> or <code>GiraphGraphComputer</code> (both are provided in Apache's distribution
+ system with an existing <code>GraphComputer</code> implementation such as, for example, <code>SparkGraphComputer</code> (provided in Apache's distribution
of TinkerPop).
</li>
</ol>
@@ -109,11 +109,7 @@ limitations under the License.
into a Big(Graph)Data processor via the OLAP component of the Gremlin traversal machine. Users do not have to learn Spark's data processing language as Gremlin traversals execute
over Spark. For graph system providers, they can boast Spark integration once a custom <code>InputRDD</code> (or <code>InputFormat</code>) is developed.
</li>
- <li><strong>GiraphGraphComputer</strong>: <a href="http://giraph.apache.org/">Apache Giraph</a>™ is a Big(Graph)Data processor that leverages <a href="http://hadoop.apache.org/">Apache Hadoop</a>® and
- <a href="http://zookeeper.apache.org/">Apache ZooKeeper</a>™ for executing distributed graph algorithms. <a href="http://tinkerpop.apache.org/docs/current/reference/#giraphgraphcomputer"><code>GiraphGraphComputer</code></a>
- supports Gremlin OLAP and allows users to submit Gremlin traversals to Giraph for distributed execution. Providers can immediately advertise Giraph integration once a custom <code>InputFormat</code> is developed.
- </li>
- <li><strong>Hadoop support</strong>: <a href="http://hadoop.apache.org/">Apache Hadoop</a>® has become a staple technology for Big Data applications. In TinkerPop, both <code>SparkGraphComputer</code> and <code>GiraphGraphComputer</code> can pull data
+ <li><strong>Hadoop support</strong>: <a href="http://hadoop.apache.org/">Apache Hadoop</a>® has become a staple technology for Big Data applications. In TinkerPop, <code>SparkGraphComputer</code> can pull data
from the Hadoop File System (<a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html">HDFS</a>). TinkerPop provides a collection of Input- and OutputFormats for different graph serialization standards as well as tooling that makes it easy for users
to <a href="http://tinkerpop.apache.org/docs/current/reference/#interacting-with-hdfs">interact with HDFS</a> from the Gremlin Console or their application.
</li>
@@ -155,7 +151,7 @@ limitations under the License.
</div>
<div class="col-sm-6 col-md-6">
<a href="http://janusgraph.org/"><img src="images/logos/janusgraph-logo.png" style="padding-right:20px;float:left;width:35%;"></a>
- <a href="http://janusgraph.org/">JanusGraph</a> is an Apache2 licensed scalable, distributed graph database optimized for storing and querying graphs containing hundreds of billions of vertices and edges distributed across a multi-machine cluster. JanusGraph is a transactional database that can support thousands of concurrent users executing complex Gremlin traversals in real time. JanusGraph also provides an in-memory, compression-based OLAP processor as well as integrates with Apache TinkerPop's Spark/Giraph OLAP processors.
+ <a href="http://janusgraph.org/">JanusGraph</a> is an Apache2 licensed scalable, distributed graph database optimized for storing and querying graphs containing hundreds of billions of vertices and edges distributed across a multi-machine cluster. JanusGraph is a transactional database that can support thousands of concurrent users executing complex Gremlin traversals in real time. JanusGraph also provides an in-memory, compression-based OLAP processor as well as integrates with Apache TinkerPop's Spark OLAP processors.
</div>
</div>
<br/>
@@ -188,7 +184,7 @@ limitations under the License.
</div>
<div class="col-sm-6 col-md-6">
<a href="http://titan.thinkaurelius.com/"><img src="images/logos/titan-logo.png" style="padding-right:20px;float:left;width:35%;"></a>
- <a href="http://titan.thinkaurelius.com/">Titan</a>™ is an Apache2 licensed scalable, distributed graph database optimized for storing and querying graphs containing hundreds of billions of vertices and edges distributed across a multi-machine cluster. Titan is a transactional database that can support thousands of concurrent users executing complex Gremlin traversals in real time. Titan also provides an in-memory, compression-based OLAP processor as well as integrates with Apache TinkerPop's Spark/Giraph OLAP processors.
+ <a href="http://titan.thinkaurelius.com/">Titan</a>™ is an Apache2 licensed scalable, distributed graph database optimized for storing and querying graphs containing hundreds of billions of vertices and edges distributed across a multi-machine cluster. Titan is a transactional database that can support thousands of concurrent users executing complex Gremlin traversals in real time. Titan also provides an in-memory, compression-based OLAP processor as well as integrates with Apache TinkerPop's Spark OLAP processors.
</div>
</div>
<br/>
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/dev/provider/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/dev/provider/index.asciidoc b/docs/src/dev/provider/index.asciidoc
index ce4e9f4..d1ef575 100644
--- a/docs/src/dev/provider/index.asciidoc
+++ b/docs/src/dev/provider/index.asciidoc
@@ -99,9 +99,9 @@ implementation is presented below:
NOTE: The VertexProgram and MapReduce interfaces in the `process/computer/` package are not required by the graph
system. Instead, these are interfaces to be implemented by application developers writing VertexPrograms and MapReduce jobs.
-IMPORTANT: TinkerPop3 provides three OLAP implementations:
+IMPORTANT: TinkerPop provides two OLAP implementations:
link:http://tinkerpop.apache.org/docs/x.y.z/reference/#tinkergraph-gremlin[TinkerGraphComputer] (TinkerGraph),
-link:http://tinkerpop.apache.org/docs/x.y.z/reference/#giraphgraphcomputer[GiraphGraphComputer] (HadoopGraph), and
+and
link:http://tinkerpop.apache.org/docs/x.y.z/reference/#sparkgraphcomputer[SparkGraphComputer] (Hadoop).
Given the complexity of the OLAP system, it is good to study and copy many of the patterns used in these reference
implementations.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/reference/gremlin-applications.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/gremlin-applications.asciidoc b/docs/src/reference/gremlin-applications.asciidoc
index 53a1642..3f6a822 100644
--- a/docs/src/reference/gremlin-applications.asciidoc
+++ b/docs/src/reference/gremlin-applications.asciidoc
@@ -2147,15 +2147,6 @@ Gephi plugin configuration parameters as accepted via the `:remote config` comma
NOTE: This plugin is typically only useful to the Gremlin Console and is enabled in the there by default.
-[[giraph-plugin]]
-=== Giraph Plugin
-
-image:giraph-logo.png[width=50,float=left] The Giraph Plugin installs as part of `giraph-gremlin` and provides
-a number of imports and utility functions to the environment within which it is used. Those classes and functions
-provide the basis for supporting <<graphcomputer,OLAP based traversals>> using link:http://giraph.apache.org[Giraph].
-This plugin is defined in greater detail in the <<giraphgraphcomputer,GiraphGraphComputer>> section and is typically
-installed in conjuction with the <<hadoop-plugin,Hadoop-Plugin>>.
-
[[graph-plugins]]
=== Graph Plugins
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/reference/implementations-giraph.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-giraph.asciidoc b/docs/src/reference/implementations-giraph.asciidoc
deleted file mode 100644
index f83903d..0000000
--- a/docs/src/reference/implementations-giraph.asciidoc
+++ /dev/null
@@ -1,145 +0,0 @@
-////
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-////
-[[giraphgraphcomputer]]
-==== GiraphGraphComputer
-
-[source,xml]
-----
-<dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>giraph-gremlin</artifactId>
- <version>x.y.z</version>
-</dependency>
-----
-
-image:giraph-logo.png[width=100,float=left] link:http://giraph.apache.org[Giraph] is an Apache Software Foundation
-project focused on OLAP-based graph processing. Giraph makes use of the distributed graph computing paradigm made
-popular by Google's Pregel. In Giraph, developers write "vertex programs" that get executed at each vertex in
-parallel. These programs communicate with one another in a bulk synchronous parallel (BSP) manner. This model aligns
-with TinkerPop3's `GraphComputer` API. TinkerPop3 provides an implementation of `GraphComputer` that works for Giraph
-called `GiraphGraphComputer`. Moreover, with TinkerPop3's <<mapreduce,MapReduce>>-framework, the standard
-Giraph/Pregel model is extended to support an arbitrary number of MapReduce phases to aggregate and yield results
-from the graph. Below are examples using `GiraphGraphComputer` from the <<gremlin-console,Gremlin-Console>>.
-
-WARNING: Giraph uses a large number of Hadoop counters. The default for Hadoop is 120. In `mapred-site.xml` it is
-possible to increase the limit it via the `mapreduce.job.counters.max` property. A good value to use is 1000. This
-is a cluster-wide property so be sure to restart the cluster after updating.
-
-WARNING: The maximum number of workers can be no larger than the number of map-slots in the Hadoop cluster minus 1.
-For example, if the Hadoop cluster has 4 map slots, then `giraph.maxWorkers` can not be larger than 3. One map-slot
-is reserved for the master compute node and all other slots can be allocated as workers to execute the VertexPrograms
-on the vertices of the graph.
-
-If `GiraphGraphComputer` will be used as the `GraphComputer` for `HadoopGraph` then its `lib` directory should be
-specified in `HADOOP_GREMLIN_LIBS`.
-
-[source,shell]
-export HADOOP_GREMLIN_LIBS=$HADOOP_GREMLIN_LIBS:/usr/local/gremlin-console/ext/giraph-gremlin/lib
-
-Or, the user can specify the directory in the Gremlin Console.
-
-[source,groovy]
-System.setProperty('HADOOP_GREMLIN_LIBS',System.getProperty('HADOOP_GREMLIN_LIBS') + ':' + '/usr/local/gremlin-console/ext/giraph-gremlin/lib')
-
-[gremlin-groovy]
-----
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-g = graph.traversal().withComputer(GiraphGraphComputer)
-g.V().count()
-g.V().out().out().values('name')
-----
-
-IMPORTANT: The examples above do not use lambdas (i.e. closures in Gremlin-Groovy). This makes the traversal
-serializable and thus, able to be distributed to all machines in the Hadoop cluster. If a lambda is required in a
-traversal, then the traversal must be sent as a `String` and compiled locally at each machine in the cluster. The
-following example demonstrates the `:remote` command which allows for submitting Gremlin traversals as a `String`.
-
-[gremlin-groovy]
-----
-graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
-g = graph.traversal().withComputer(GiraphGraphComputer)
-:remote connect tinkerpop.hadoop graph g
-:> g.V().group().by{it.value('name')[1]}.by('name')
-result
-result.memory.runtime
-----
-
-NOTE: If the user explicitly specifies `giraph.maxWorkers` and/or `giraph.numComputeThreads` in the configuration,
-then these values will be used by Giraph. However, if these are not specified and the user never calls
-`GraphComputer.workers()` then `GiraphGraphComputer` will try to compute the number of workers/threads to use based
-on the cluster's profile.
-
-===== Loading with BulkLoaderVertexProgram
-
-The <<bulkloadervertexprogram, BulkLoaderVertexProgram>> is a generalized bulk loader that can be used to load
-large amounts of data to and from different `Graph` implementations. The following code demonstrates how to load
-the Grateful Dead graph from HadoopGraph into TinkerGraph over Giraph:
-
-[gremlin-groovy]
-----
-hdfs.copyFromLocal('data/grateful-dead.kryo', 'grateful-dead.kryo')
-readGraph = GraphFactory.open('conf/hadoop/hadoop-grateful-gryo.properties')
-writeGraph = 'conf/tinkergraph-gryo.properties'
-blvp = BulkLoaderVertexProgram.build().
- keepOriginalIds(false).
- writeGraph(writeGraph).create(readGraph)
-readGraph.compute(GiraphGraphComputer).workers(1).program(blvp).submit().get()
-:set max-iteration 10
-graph = GraphFactory.open(writeGraph)
-g = graph.traversal()
-g.V().valueMap()
-graph.close()
-----
-
-[source,properties]
-----
-# hadoop-grateful-gryo.properties
-
-#
-# Hadoop Graph Configuration
-#
-gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
-gremlin.hadoop.graphReader=org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat
-gremlin.hadoop.graphWriter=org.apache.hadoop.mapreduce.lib.output.NullOutputFormat
-gremlin.hadoop.inputLocation=grateful-dead.kryo
-gremlin.hadoop.outputLocation=output
-gremlin.hadoop.jarsInDistributedCache=true
-
-#
-# GiraphGraphComputer Configuration
-#
-giraph.minWorkers=1
-giraph.maxWorkers=1
-giraph.useOutOfCoreGraph=true
-giraph.useOutOfCoreMessages=true
-mapred.map.child.java.opts=-Xmx1024m
-mapred.reduce.child.java.opts=-Xmx1024m
-giraph.numInputThreads=4
-giraph.numComputeThreads=4
-giraph.maxMessagesInMemory=100000
-----
-
-[source,properties]
-----
-# tinkergraph-gryo.properties
-
-gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
-gremlin.tinkergraph.graphFormat=gryo
-gremlin.tinkergraph.graphLocation=/tmp/tinkergraph.kryo
-----
-
-NOTE: The path to TinkerGraph needs to be included in the `HADOOP_GREMLIN_LIBS` for the above example to work.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/reference/implementations-hadoop-end.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-hadoop-end.asciidoc b/docs/src/reference/implementations-hadoop-end.asciidoc
index f0f0fa6..e11c542 100644
--- a/docs/src/reference/implementations-hadoop-end.asciidoc
+++ b/docs/src/reference/implementations-hadoop-end.asciidoc
@@ -170,7 +170,7 @@ hdfs.ls()
==== Interacting with Spark
If a Spark context is persisted, then Spark RDDs will remain the Spark cache and accessible over subsequent jobs.
-RDDs are retrieved and saved to the `SparkContext` via `PersistedInputRDD` and `PersistedOutputRDD` respectivly.
+RDDs are retrieved and saved to the `SparkContext` via `PersistedInputRDD` and `PersistedOutputRDD` respectively.
Persisted RDDs can be accessed using `spark`.
[gremlin-groovy]
@@ -186,144 +186,4 @@ spark.head('output', PersistedInputRDD)
spark.head('output', 'clusterCount', PersistedInputRDD)
spark.rm('output')
spark.ls()
-----
-
-=== A Command Line Example
-
-image::pagerank-logo.png[width=300]
-
-The classic link:http://en.wikipedia.org/wiki/PageRank[PageRank] centrality algorithm can be executed over the
-TinkerPop graph from the command line using `GiraphGraphComputer`.
-
-WARNING: Be sure that the `HADOOP_GREMLIN_LIBS` references the location `lib` directory of the respective
-`GraphComputer` engine being used or else the requisite dependencies will not be uploaded to the Hadoop cluster.
-
-[source,text]
-----
-$ hdfs dfs -copyFromLocal data/tinkerpop-modern.json tinkerpop-modern.json
-$ hdfs dfs -ls
-Found 2 items
--rw-r--r-- 1 marko supergroup 2356 2014-07-28 13:00 /user/marko/tinkerpop-modern.json
-$ hadoop jar target/giraph-gremlin-x.y.z-job.jar org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer ../hadoop-gremlin/conf/hadoop-graphson.properties
-15/09/11 08:02:08 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
-15/09/11 08:02:11 INFO computer.GiraphGraphComputer: HadoopGremlin(Giraph): PageRankVertexProgram[alpha=0.85,iterations=30]
-15/09/11 08:02:12 INFO mapreduce.JobSubmitter: number of splits:3
-15/09/11 08:02:12 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1441915907347_0028
-15/09/11 08:02:12 INFO impl.YarnClientImpl: Submitted application application_1441915907347_0028
-15/09/11 08:02:12 INFO job.GiraphJob: Tracking URL: http://markos-macbook:8088/proxy/application_1441915907347_0028/
-15/09/11 08:02:12 INFO job.GiraphJob: Waiting for resources... Job will start only when it gets all 3 mappers
-15/09/11 08:03:54 INFO mapreduce.Job: Running job: job_1441915907347_0028
-15/09/11 08:03:55 INFO mapreduce.Job: Job job_1441915907347_0028 running in uber mode : false
-15/09/11 08:03:55 INFO mapreduce.Job: map 33% reduce 0%
-15/09/11 08:03:57 INFO mapreduce.Job: map 67% reduce 0%
-15/09/11 08:04:01 INFO mapreduce.Job: map 100% reduce 0%
-15/09/11 08:06:17 INFO mapreduce.Job: Job job_1441915907347_0028 completed successfully
-15/09/11 08:06:17 INFO mapreduce.Job: Counters: 80
- File System Counters
- FILE: Number of bytes read=0
- FILE: Number of bytes written=483918
- FILE: Number of read operations=0
- FILE: Number of large read operations=0
- FILE: Number of write operations=0
- HDFS: Number of bytes read=1465
- HDFS: Number of bytes written=1760
- HDFS: Number of read operations=39
- HDFS: Number of large read operations=0
- HDFS: Number of write operations=20
- Job Counters
- Launched map tasks=3
- Other local map tasks=3
- Total time spent by all maps in occupied slots (ms)=458105
- Total time spent by all reduces in occupied slots (ms)=0
- Total time spent by all map tasks (ms)=458105
- Total vcore-seconds taken by all map tasks=458105
- Total megabyte-seconds taken by all map tasks=469099520
- Map-Reduce Framework
- Map input records=3
- Map output records=0
- Input split bytes=132
- Spilled Records=0
- Failed Shuffles=0
- Merged Map outputs=0
- GC time elapsed (ms)=1594
- CPU time spent (ms)=0
- Physical memory (bytes) snapshot=0
- Virtual memory (bytes) snapshot=0
- Total committed heap usage (bytes)=527958016
- Giraph Stats
- Aggregate edges=0
- Aggregate finished vertices=0
- Aggregate sent message message bytes=13535
- Aggregate sent messages=186
- Aggregate vertices=6
- Current master task partition=0
- Current workers=2
- Last checkpointed superstep=0
- Sent message bytes=438
- Sent messages=6
- Superstep=31
- Giraph Timers
- Initialize (ms)=2996
- Input superstep (ms)=5209
- Setup (ms)=59
- Shutdown (ms)=9324
- Superstep 0 GiraphComputation (ms)=3861
- Superstep 1 GiraphComputation (ms)=4027
- Superstep 10 GiraphComputation (ms)=4000
- Superstep 11 GiraphComputation (ms)=4004
- Superstep 12 GiraphComputation (ms)=3999
- Superstep 13 GiraphComputation (ms)=4000
- Superstep 14 GiraphComputation (ms)=4005
- Superstep 15 GiraphComputation (ms)=4003
- Superstep 16 GiraphComputation (ms)=4001
- Superstep 17 GiraphComputation (ms)=4007
- Superstep 18 GiraphComputation (ms)=3998
- Superstep 19 GiraphComputation (ms)=4006
- Superstep 2 GiraphComputation (ms)=4007
- Superstep 20 GiraphComputation (ms)=3996
- Superstep 21 GiraphComputation (ms)=4006
- Superstep 22 GiraphComputation (ms)=4002
- Superstep 23 GiraphComputation (ms)=3998
- Superstep 24 GiraphComputation (ms)=4003
- Superstep 25 GiraphComputation (ms)=4001
- Superstep 26 GiraphComputation (ms)=4003
- Superstep 27 GiraphComputation (ms)=4005
- Superstep 28 GiraphComputation (ms)=4002
- Superstep 29 GiraphComputation (ms)=4001
- Superstep 3 GiraphComputation (ms)=3988
- Superstep 30 GiraphComputation (ms)=4248
- Superstep 4 GiraphComputation (ms)=4010
- Superstep 5 GiraphComputation (ms)=3998
- Superstep 6 GiraphComputation (ms)=3996
- Superstep 7 GiraphComputation (ms)=4005
- Superstep 8 GiraphComputation (ms)=4009
- Superstep 9 GiraphComputation (ms)=3994
- Total (ms)=138788
- File Input Format Counters
- Bytes Read=0
- File Output Format Counters
- Bytes Written=0
-$ hdfs dfs -cat output/~g/*
-{"id":1,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.15000000000000002}],"name":[{"id":0,"value":"marko"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":3.0}],"age":[{"id":1,"value":29}]}}
-{"id":5,"label":"software","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":35,"value":0.23181250000000003}],"name":[{"id":8,"value":"ripple"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":6,"value":0.0}],"lang":[{"id":9,"value":"java"}]}}
-{"id":3,"label":"software","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.4018125}],"name":[{"id":4,"value":"lop"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":0.0}],"lang":[{"id":5,"value":"java"}]}}
-{"id":4,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.19250000000000003}],"name":[{"id":6,"value":"josh"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":2.0}],"age":[{"id":7,"value":32}]}}
-{"id":2,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":35,"value":0.19250000000000003}],"name":[{"id":2,"value":"vadas"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":6,"value":0.0}],"age":[{"id":3,"value":27}]}}
-{"id":6,"label":"person","properties":{"gremlin.pageRankVertexProgram.pageRank":[{"id":35,"value":0.15000000000000002}],"name":[{"id":10,"value":"peter"}],"gremlin.pageRankVertexProgram.edgeCount":[{"id":6,"value":1.0}],"age":[{"id":11,"value":35}]}}
-----
-
-Vertex 4 ("josh") is isolated below:
-
-[source,js]
-----
-{
- "id":4,
- "label":"person",
- "properties": {
- "gremlin.pageRankVertexProgram.pageRank":[{"id":39,"value":0.19250000000000003}],
- "name":[{"id":6,"value":"josh"}],
- "gremlin.pageRankVertexProgram.edgeCount":[{"id":10,"value":2.0}],
- "age":[{"id":7,"value":32}]}
- }
-}
-----
+----
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/reference/implementations-hadoop-start.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations-hadoop-start.asciidoc b/docs/src/reference/implementations-hadoop-start.asciidoc
index 31ecf6b..8825009 100644
--- a/docs/src/reference/implementations-hadoop-start.asciidoc
+++ b/docs/src/reference/implementations-hadoop-start.asciidoc
@@ -34,9 +34,8 @@ using both TinkerPop3's OLTP and OLAP graph computing models.
IMPORTANT: This section assumes that the user has a Hadoop 2.x cluster functioning. For more information on getting
started with Hadoop, please see the
link:http://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/SingleCluster.html[Single Node Setup]
-tutorial. Moreover, if using `GiraphGraphComputer` or `SparkGraphComputer` it is advisable that the reader also
-familiarize their self with Giraph (link:http://giraph.apache.org/quick_start.html[Getting Started]) and Spark
-(link:http://spark.apache.org/docs/latest/quick-start.html[Quick Start]).
+tutorial. Moreover, if using `SparkGraphComputer` it is advisable that the reader also
+familiarize their self with and Spark (link:http://spark.apache.org/docs/latest/quick-start.html[Quick Start]).
=== Installing Hadoop-Gremlin
@@ -87,11 +86,7 @@ distributed cache (link:http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop
Note that the locations in `HADOOP_GREMLIN_LIBS` can be colon-separated (`:`) and all jars from all locations will
be loaded into the cluster. Locations can be local paths (e.g. `/path/to/libs`), but may also be prefixed with a file
scheme to reference files or directories in different file systems (e.g. `hdfs:///path/to/distributed/libs`).
-Typically, only the jars of the respective GraphComputer are required to be loaded (e.g. `GiraphGraphComputer` plugin lib
-directory).
-
-[source,shell]
-export HADOOP_GREMLIN_LIBS=/usr/local/gremlin-console/ext/giraph-gremlin/lib
+Typically, only the jars of the respective `GraphComputer` are required to be loaded.
=== Properties Files
@@ -113,21 +108,9 @@ spark.master=local[4]
spark.executor.memory=1g
spark.serializer=org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer
gremlin.spark.persistContext=true
-#####################################
-# GiraphGraphComputer Configuration #
-#####################################
-giraph.minWorkers=2
-giraph.maxWorkers=2
-giraph.useOutOfCoreGraph=true
-giraph.useOutOfCoreMessages=true
-mapreduce.map.java.opts=-Xmx1024m
-mapreduce.reduce.java.opts=-Xmx1024m
-giraph.numInputThreads=2
-giraph.numComputeThreads=2
A review of the Hadoop-Gremlin specific properties are provided in the table below. For the respective OLAP
-engines (<<sparkgraphcomputer,`SparkGraphComputer`>> or <<giraphgraphcomputer,`GiraphGraphComputer`>>) refer
-to their respective documentation for configuration options.
+engines (<<sparkgraphcomputer,`SparkGraphComputer`>> refer to their respective documentation for configuration options.
[width="100%",cols="2,10",options="header"]
|=========================================================
@@ -145,7 +128,7 @@ Along with the properties above, the numerous link:http://hadoop.apache.org/docs
can be added as needed to tune and parameterize the executed Hadoop-Gremlin job on the respective Hadoop cluster.
IMPORTANT: As the size of the graphs being processed becomes large, it is important to fully understand how the
-underlying OLAP engine (e.g. Spark, Giraph, etc.) works and understand the numerous parameterizations offered by
+underlying OLAP engine (e.g. Spark, etc.) works and understand the numerous parameterizations offered by
these systems. Such knowledge can help alleviate out of memory exceptions, slow load times, slow processing times,
garbage collection issues, etc.
@@ -183,17 +166,11 @@ supports the following two implementations.
* <<sparkgraphcomputer,`SparkGraphComputer`>>: Leverages Apache Spark to execute TinkerPop3 OLAP computations.
** The graph may fit within the total RAM of the cluster (supports larger graphs). Message passing is coordinated via
Spark map/reduce/join operations on in-memory and disk-cached data (average speed traversals).
-* <<giraphgraphcomputer,`GiraphGraphComputer`>>: Leverages Apache Giraph to execute TinkerPop3 OLAP computations.
-** The graph should fit within the total RAM of the Hadoop cluster (graph size restriction), though "out-of-core"
-processing is possible. Message passing is coordinated via ZooKeeper for the in-memory graph (speedy traversals).
TIP: image:gremlin-sugar.png[width=50,float=left] For those wanting to use the <<sugar-plugin,SugarPlugin>> with
their submitted traversal, do `:remote config useSugar true` as well as `:plugin use tinkerpop.sugar` at the start of
the Gremlin Console session if it is not already activated.
-Note that `SparkGraphComputer` and `GiraphGraphComputer` are loaded via their respective plugins. Typically only
-one plugin or the other is loaded depending on the desired `GraphComputer` to use.
-
[source,text]
----
$ bin/gremlin.sh
@@ -205,8 +182,6 @@ plugin activated: tinkerpop.server
plugin activated: tinkerpop.utilities
plugin activated: tinkerpop.tinkergraph
plugin activated: tinkerpop.hadoop
-gremlin> :install org.apache.tinkerpop giraph-gremlin x.y.z
-==>loaded: [org.apache.tinkerpop, giraph-gremlin, x.y.z] - restart the console to use [tinkerpop.giraph]
gremlin> :install org.apache.tinkerpop spark-gremlin x.y.z
==>loaded: [org.apache.tinkerpop, spark-gremlin, x.y.z] - restart the console to use [tinkerpop.spark]
gremlin> :q
@@ -219,13 +194,10 @@ plugin activated: tinkerpop.server
plugin activated: tinkerpop.utilities
plugin activated: tinkerpop.tinkergraph
plugin activated: tinkerpop.hadoop
-gremlin> :plugin use tinkerpop.giraph
-==>tinkerpop.giraph activated
gremlin> :plugin use tinkerpop.spark
==>tinkerpop.spark activated
----
-WARNING: Hadoop, Spark, and Giraph all depend on many of the same libraries (e.g. ZooKeeper, Snappy, Netty, Guava,
+WARNING: Hadoop and Spark all depend on many of the same libraries (e.g. ZooKeeper, Snappy, Netty, Guava,
etc.). Unfortunately, typically these dependencies are not to the same versions of the respective libraries. As such,
-it is best to *not* have both Spark and Giraph plugins loaded in the same console session nor in the same Java
-project (though intelligent `<exclusion>`-usage can help alleviate conflicts in a Java project).
+it is may be necessary to manually cleanup dependency conflicts among different plugins.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/reference/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/index.asciidoc b/docs/src/reference/index.asciidoc
index decc7ef..6e81bf2 100644
--- a/docs/src/reference/index.asciidoc
+++ b/docs/src/reference/index.asciidoc
@@ -41,7 +41,6 @@ include::implementations-neo4j.asciidoc[]
// console is to have a new asciidoc page.
include::implementations-hadoop-start.asciidoc[]
include::implementations-spark.asciidoc[]
-include::implementations-giraph.asciidoc[]
include::implementations-hadoop-end.asciidoc[]
include::gremlin-variants.asciidoc[]
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/tutorials/getting-started/index.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/tutorials/getting-started/index.asciidoc b/docs/src/tutorials/getting-started/index.asciidoc
index af3aebc..622c596 100644
--- a/docs/src/tutorials/getting-started/index.asciidoc
+++ b/docs/src/tutorials/getting-started/index.asciidoc
@@ -465,13 +465,13 @@ implementations are behind TinkerPop APIs, which open the possibility to switch
TinkerPop has always had the vision of being an abstraction over different graph databases. That much
is not new and dates back to TinkerPop 1.x. It is in TinkerPop 3.x however that we see the introduction of the notion
-that TinkerPop is also an abstraction over different graph processors like link:http://spark.apache.org[Spark] and
-link:http://giraph.apache.org/[Giraph]. The scope of this tutorial does not permit it to delve into
-"graph processors", but the short story is that the same Gremlin statement we wrote in the examples above can be
-executed to run in distributed fashion over Spark or Hadoop. The changes required to the code to do this are not
-in the traversal itself, but in the definition of the `TraversalSource`. You can again see why we encourage, graph
-operations to be executed through that class as opposed to just using `Graph`. You can read more about these
-features in this section on link:http://tinkerpop.apache.org/docs/x.y.z/reference/#hadoop-gremlin[hadoop-gremlin].
+that TinkerPop is also an abstraction over different graph processors like link:http://spark.apache.org[Spark]. The
+scope of this tutorial does not permit it to delve into "graph processors", but the short story is that the same
+Gremlin statement we wrote in the examples above can be executed to run in distributed fashion over Spark or Hadoop.
+The changes required to the code to do this are not in the traversal itself, but in the definition of the
+`TraversalSource`. You can again see why we encourage, graph operations to be executed through that class as opposed
+to just using `Graph`. You can read more about these features in this section on
+link:http://tinkerpop.apache.org/docs/x.y.z/reference/#hadoop-gremlin[hadoop-gremlin].
TIP: To maintain an abstraction over `Graph` creation use `GraphFactory.open()` to construct new instances. See
the documentation for individual `Graph` implementations to learn about the configuration options to provide.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/docs/src/upgrade/release-3.4.x.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/upgrade/release-3.4.x.asciidoc b/docs/src/upgrade/release-3.4.x.asciidoc
index 1916da8..713916a 100644
--- a/docs/src/upgrade/release-3.4.x.asciidoc
+++ b/docs/src/upgrade/release-3.4.x.asciidoc
@@ -29,6 +29,14 @@ Please see the link:https://github.com/apache/tinkerpop/blob/3.4.0/CHANGELOG.asc
=== Upgrading for Users
+==== Removal of Giraph Support
+
+Support for Giraph has been removed as of this version. There were a number of reasons for this decision which were
+discussed in the community prior to taking this step. Users should switch to Spark for their OLAP based graph-computing
+needs.
+
+See: link:https://issues.apache.org/jira/browse/TINKERPOP-1930[TINKERPOP-1930]
+
==== Modifications to reducing barrier steps
The behavior of `min()`, `max()`, `mean()` and `sum()` has been modified to return no result if there's no input. Previously these steps yielded the internal seed value:
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml
deleted file mode 100644
index 8cf3f5b..0000000
--- a/giraph-gremlin/pom.xml
+++ /dev/null
@@ -1,258 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>tinkerpop</artifactId>
- <version>3.4.0-SNAPSHOT</version>
- </parent>
- <artifactId>giraph-gremlin</artifactId>
- <name>Apache TinkerPop :: Giraph Gremlin</name>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <!-- see: https://github.com/apache/hadoop/pull/84 -->
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.7.1.Final</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>gremlin-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>hadoop-gremlin</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- GIRAPH -->
- <dependency>
- <groupId>org.apache.giraph</groupId>
- <artifactId>giraph-core</artifactId>
- <version>1.1.0-hadoop2</version>
- <exclusions>
- <!-- self conflicts -->
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <!-- conflicts with hadoop-gremlin -->
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
- <!-- uses an older version that conflicts with spark 2.0 -->
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <!-- conflicts with gremlin-groovy -->
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- <!-- conflicts with gremlin-core -->
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <!-- gremlin-groovy conflicts -->
- <exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </exclusion>
- <!-- conflicts with spark (a hack so both can be loaded in parallel) -->
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- consistent dependencies -->
- <dependency>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- <version>3.1.0</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <version>2.6.5</version>
- </dependency>
- <!-- TEST -->
- <dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>gremlin-test</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>hadoop-gremlin</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.tinkerpop</groupId>
- <artifactId>tinkergraph-gremlin</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <directory>${basedir}/target</directory>
- <finalName>${project.artifactId}-${project.version}</finalName>
- <resources>
- <resource>
- <directory>${basedir}/src/main/resources</directory>
- </resource>
- </resources>
- <testResources>
- <testResource>
- <directory>${basedir}/src/test/resources</directory>
- </testResource>
- </testResources>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>build-detached-assemblies</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <attach>false</attach>
- <descriptors>
- <descriptor>src/assembly/standalone.xml</descriptor>
- <descriptor>src/assembly/hadoop-job.xml</descriptor>
- </descriptors>
- <finalName>${project.artifactId}-${project.version}</finalName>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-failsafe-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.gmavenplus</groupId>
- <artifactId>gmavenplus-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>addSources</goal>
- <goal>addTestSources</goal>
- <goal>generateStubs</goal>
- <goal>compile</goal>
- <goal>generateTestStubs</goal>
- <goal>compileTests</goal>
- <goal>removeStubs</goal>
- <goal>removeTestStubs</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <Gremlin-Plugin-Dependencies>org.apache.hadoop:hadoop-client:${hadoop.version}
- </Gremlin-Plugin-Dependencies>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/assembly/hadoop-job.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/assembly/hadoop-job.xml b/giraph-gremlin/src/assembly/hadoop-job.xml
deleted file mode 100644
index 65233e9..0000000
--- a/giraph-gremlin/src/assembly/hadoop-job.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<assembly>
- <id>job</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <unpack>false</unpack>
- <scope>runtime</scope>
- <outputDirectory>lib</outputDirectory>
- <excludes>
- <exclude>${groupId}:${artifactId}</exclude>
- </excludes>
- </dependencySet>
- <dependencySet>
- <unpack>true</unpack>
- <includes>
- <include>${groupId}:${artifactId}</include>
- </includes>
- </dependencySet>
- </dependencySets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/assembly/standalone.xml
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/assembly/standalone.xml b/giraph-gremlin/src/assembly/standalone.xml
deleted file mode 100644
index 82b8514..0000000
--- a/giraph-gremlin/src/assembly/standalone.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<assembly>
- <id>standalone</id>
- <formats>
- <format>dir</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
-
- <fileSets>
- <fileSet>
- <directory>src/main/bin</directory>
- <outputDirectory>/bin</outputDirectory>
- <fileMode>0755</fileMode>
- </fileSet>
- <fileSet>
- <directory>target/*.jar</directory>
- <outputDirectory>/lib</outputDirectory>
- </fileSet>
- </fileSets>
-
- <dependencySets>
- <dependencySet>
- <outputDirectory>/lib</outputDirectory>
- <unpack>false</unpack>
- <scope>compile</scope>
- </dependencySet>
- <dependencySet>
- <outputDirectory>/lib</outputDirectory>
- <unpack>false</unpack>
- <scope>provided</scope>
- </dependencySet>
- </dependencySets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/jsr223/GiraphGremlinPlugin.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/jsr223/GiraphGremlinPlugin.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/jsr223/GiraphGremlinPlugin.java
deleted file mode 100644
index c512335..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/jsr223/GiraphGremlinPlugin.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.jsr223;
-
-import org.apache.tinkerpop.gremlin.giraph.process.computer.EmptyOutEdges;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphComputation;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphMemory;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphMessageCombiner;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphMessenger;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphVertex;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphWorkerContext;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.MemoryAggregator;
-import org.apache.tinkerpop.gremlin.giraph.process.computer.PassThroughMemory;
-import org.apache.tinkerpop.gremlin.jsr223.AbstractGremlinPlugin;
-import org.apache.tinkerpop.gremlin.jsr223.DefaultImportCustomizer;
-
-/**
- * @author Stephen Mallette (http://stephen.genoprime.com)
- */
-public final class GiraphGremlinPlugin extends AbstractGremlinPlugin {
- private static final String MODULE_NAME = "tinkerpop.giraph";
- private static final GiraphGremlinPlugin instance = new GiraphGremlinPlugin();
-
- public GiraphGremlinPlugin() {
- super(MODULE_NAME, DefaultImportCustomizer.build().addClassImports(
- EmptyOutEdges.class,
- GiraphComputation.class,
- GiraphGraphComputer.class,
- GiraphMemory.class,
- GiraphMessageCombiner.class,
- GiraphMessenger.class,
- GiraphVertex.class,
- GiraphWorkerContext.class,
- MemoryAggregator.class,
- PassThroughMemory.class).create());
- }
-
- public static GiraphGremlinPlugin instance() {
- return instance;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java
deleted file mode 100644
index 4df4835..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/EmptyOutEdges.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.giraph.edge.Edge;
-import org.apache.giraph.edge.OutEdges;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class EmptyOutEdges implements OutEdges<ObjectWritable, NullWritable> {
-
- private static final EmptyOutEdges INSTANCE = new EmptyOutEdges();
-
- public static EmptyOutEdges instance() {
- return INSTANCE;
- }
-
- @Override
- public void initialize(final Iterable<Edge<ObjectWritable, NullWritable>> edges) {
- }
-
- @Override
- public void initialize(final int capacity) {
- }
-
- @Override
- public void initialize() {
- }
-
- @Override
- public void add(final Edge<ObjectWritable, NullWritable> edge) {
- }
-
- @Override
- public void remove(final ObjectWritable targetVertexId) {
- }
-
- @Override
- public int size() {
- return 0;
- }
-
- @Override
- public Iterator<Edge<ObjectWritable, NullWritable>> iterator() {
- return Collections.emptyIterator();
- }
-
- @Override
- public void write(final DataOutput dataOutput) throws IOException {
- }
-
- @Override
- public void readFields(final DataInput dataInput) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java
deleted file mode 100644
index 1d52566..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphComputation.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.giraph.graph.BasicComputation;
-import org.apache.giraph.graph.Vertex;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
-
-import java.io.IOException;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphComputation extends BasicComputation<ObjectWritable, VertexWritable, NullWritable, ObjectWritable> {
-
- public GiraphComputation() {
- }
-
- @Override
- public void compute(final Vertex<ObjectWritable, VertexWritable, NullWritable> vertex, final Iterable<ObjectWritable> messages) throws IOException {
- final GiraphWorkerContext workerContext = this.getWorkerContext();
- final VertexProgram<?> vertexProgram = workerContext.getVertexProgramPool().take();
- vertexProgram.execute(ComputerGraph.vertexProgram(vertex.getValue().get(), vertexProgram), workerContext.getMessenger((GiraphVertex) vertex, this, messages.iterator()), workerContext.getMemory());
- workerContext.getVertexProgramPool().offer(vertexProgram);
- }
-}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ed5924c3/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
----------------------------------------------------------------------
diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
deleted file mode 100644
index 1be548a..0000000
--- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/process/computer/GiraphGraphComputer.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.giraph.process.computer;
-
-import org.apache.commons.configuration.BaseConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.FileConfiguration;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.giraph.conf.GiraphConfiguration;
-import org.apache.giraph.conf.GiraphConstants;
-import org.apache.giraph.job.GiraphJob;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.Cluster;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.tinkerpop.gremlin.giraph.structure.io.GiraphVertexInputFormat;
-import org.apache.tinkerpop.gremlin.giraph.structure.io.GiraphVertexOutputFormat;
-import org.apache.tinkerpop.gremlin.hadoop.Constants;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.ComputerSubmissionHelper;
-import org.apache.tinkerpop.gremlin.hadoop.process.computer.util.MapReduceHelper;
-import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorage;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.GraphFilterAware;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPoolShimService;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.InputOutputHelper;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
-import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
-import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
-import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
-import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
-import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
-import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
-import org.apache.tinkerpop.gremlin.structure.io.Storage;
-import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader;
-import org.apache.tinkerpop.gremlin.util.Gremlin;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.NotSerializableException;
-import java.net.URI;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Future;
-
-/**
- * @author Marko A. Rodriguez (http://markorodriguez.com)
- */
-public final class GiraphGraphComputer extends AbstractHadoopGraphComputer implements GraphComputer, Tool {
-
- protected GiraphConfiguration giraphConfiguration = new GiraphConfiguration();
- private MapMemory memory = new MapMemory();
- private boolean useWorkerThreadsInConfiguration;
- private Set<String> vertexProgramConfigurationKeys = new HashSet<>();
-
- public GiraphGraphComputer(final HadoopGraph hadoopGraph) {
- super(hadoopGraph);
- final Configuration configuration = hadoopGraph.configuration();
- configuration.getKeys().forEachRemaining(key -> this.giraphConfiguration.set(key, configuration.getProperty(key).toString()));
- this.giraphConfiguration.setMasterComputeClass(GiraphMemory.class);
- this.giraphConfiguration.setVertexClass(GiraphVertex.class);
- this.giraphConfiguration.setComputationClass(GiraphComputation.class);
- this.giraphConfiguration.setWorkerContextClass(GiraphWorkerContext.class);
- this.giraphConfiguration.setOutEdgesClass(EmptyOutEdges.class);
- this.giraphConfiguration.setClass(GiraphConstants.VERTEX_ID_CLASS.getKey(), ObjectWritable.class, ObjectWritable.class);
- this.giraphConfiguration.setClass(GiraphConstants.VERTEX_VALUE_CLASS.getKey(), VertexWritable.class, VertexWritable.class);
- this.giraphConfiguration.setBoolean(GiraphConstants.STATIC_GRAPH.getKey(), true);
- this.giraphConfiguration.setVertexInputFormatClass(GiraphVertexInputFormat.class);
- this.giraphConfiguration.setVertexOutputFormatClass(GiraphVertexOutputFormat.class);
- this.useWorkerThreadsInConfiguration = this.giraphConfiguration.getInt(GiraphConstants.MAX_WORKERS, -666) != -666 || this.giraphConfiguration.getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), -666) != -666;
- }
-
- @Override
- public GraphComputer workers(final int workers) {
- this.useWorkerThreadsInConfiguration = false;
- return super.workers(workers);
- }
-
- @Override
- public GraphComputer configure(final String key, final Object value) {
- this.giraphConfiguration.set(key, value.toString());
- this.useWorkerThreadsInConfiguration = this.giraphConfiguration.getInt(GiraphConstants.MAX_WORKERS, -666) != -666 || this.giraphConfiguration.getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), -666) != -666;
- return this;
- }
-
- @Override
- public GraphComputer program(final VertexProgram vertexProgram) {
- super.program(vertexProgram);
- this.memory.addVertexProgramMemoryComputeKeys(this.vertexProgram);
- final BaseConfiguration apacheConfiguration = new BaseConfiguration();
- apacheConfiguration.setDelimiterParsingDisabled(true);
- vertexProgram.storeState(apacheConfiguration);
- IteratorUtils.fill(apacheConfiguration.getKeys(), this.vertexProgramConfigurationKeys);
- ConfUtil.mergeApacheIntoHadoopConfiguration(apacheConfiguration, this.giraphConfiguration);
- this.vertexProgram.getMessageCombiner().ifPresent(combiner -> this.giraphConfiguration.setMessageCombinerClass(GiraphMessageCombiner.class));
- return this;
- }
-
- @Override
- public Future<ComputerResult> submit() {
- super.validateStatePriorToExecution();
- return ComputerSubmissionHelper.runWithBackgroundThread(this::submitWithExecutor, "GiraphSubmitter");
- }
-
- private Future<ComputerResult> submitWithExecutor(final Executor exec) {
- final long startTime = System.currentTimeMillis();
- final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.giraphConfiguration);
- return CompletableFuture.<ComputerResult>supplyAsync(() -> {
- try {
- this.loadJars(giraphConfiguration);
- ToolRunner.run(this, new String[]{});
- } catch (final Exception e) {
- //e.printStackTrace();
- throw new IllegalStateException(e.getMessage(), e);
- }
- this.memory.setRuntime(System.currentTimeMillis() - startTime);
- // clear properties that should not be propagated in an OLAP chain
- apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_GRAPH_FILTER);
- apacheConfiguration.clearProperty(Constants.GREMLIN_HADOOP_VERTEX_PROGRAM_INTERCEPTOR);
- this.vertexProgramConfigurationKeys.forEach(apacheConfiguration::clearProperty); // clear out vertex program specific configurations
- return new DefaultComputerResult(InputOutputHelper.getOutputGraph(apacheConfiguration, this.resultGraph, this.persist), this.memory.asImmutable());
- }, exec);
- }
-
- @Override
- public int run(final String[] args) {
- final Storage storage = FileSystemStorage.open(this.giraphConfiguration);
- storage.rm(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION));
- this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, this.persist.equals(Persist.EDGES));
- try {
- // store vertex and edge filters (will propagate down to native InputFormat or else GiraphVertexInputFormat will process)
- final BaseConfiguration apacheConfiguration = new BaseConfiguration();
- apacheConfiguration.setDelimiterParsingDisabled(true);
- GraphFilterAware.storeGraphFilter(apacheConfiguration, this.giraphConfiguration, this.graphFilter);
-
- // it is possible to run graph computer without a vertex program (and thus, only map reduce jobs if they exist)
- if (null != this.vertexProgram) {
- // a way to verify in Giraph whether the traversal will go over the wire or not
- try {
- VertexProgram.createVertexProgram(this.hadoopGraph, ConfUtil.makeApacheConfiguration(this.giraphConfiguration));
- } catch (final IllegalStateException e) {
- if (e.getCause() instanceof NumberFormatException)
- throw new NotSerializableException("The provided traversal is not serializable and thus, can not be distributed across the cluster");
- }
- // remove historic combiners in configuration propagation (this occurs when job chaining)
- if (!this.vertexProgram.getMessageCombiner().isPresent())
- this.giraphConfiguration.unset(GiraphConstants.MESSAGE_COMBINER_CLASS.getKey());
- // split required workers across system (open map slots + max threads per machine = total amount of TinkerPop workers)
- if (!this.useWorkerThreadsInConfiguration) {
- final Cluster cluster = new Cluster(GiraphGraphComputer.this.giraphConfiguration);
- int totalMappers = cluster.getClusterStatus().getMapSlotCapacity() - 1; // 1 is needed for master
- cluster.close();
- if (this.workers <= totalMappers) {
- this.giraphConfiguration.setWorkerConfiguration(this.workers, this.workers, 100.0F);
- this.giraphConfiguration.setNumComputeThreads(1);
- } else {
- if (totalMappers == 0) totalMappers = 1; // happens in local mode
- int threadsPerMapper = Long.valueOf(Math.round((double) this.workers / (double) totalMappers)).intValue(); // TODO: need to find least common denominator
- this.giraphConfiguration.setWorkerConfiguration(totalMappers, totalMappers, 100.0F);
- this.giraphConfiguration.setNumComputeThreads(threadsPerMapper);
- }
- }
- // prepare the giraph vertex-centric computing job
- final GiraphJob job = new GiraphJob(this.giraphConfiguration, Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
- job.getInternalJob().setJarByClass(GiraphGraphComputer.class);
- this.logger.info(Constants.GREMLIN_HADOOP_GIRAPH_JOB_PREFIX + this.vertexProgram);
- // handle input paths (if any)
- String inputLocation = this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION, null);
- if (null != inputLocation && FileInputFormat.class.isAssignableFrom(this.giraphConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_READER, InputFormat.class))) {
- inputLocation = Constants.getSearchGraphLocation(inputLocation, storage).orElse(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
- FileInputFormat.setInputPaths(job.getInternalJob(), new Path(inputLocation));
- }
- // handle output paths (if any)
- String outputLocation = this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, null);
- if (null != outputLocation && FileOutputFormat.class.isAssignableFrom(this.giraphConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_WRITER, OutputFormat.class))) {
- outputLocation = Constants.getGraphLocation(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION));
- FileOutputFormat.setOutputPath(job.getInternalJob(), new Path(outputLocation));
- }
- // execute the job and wait until it completes (if it fails, throw an exception)
- if (!job.run(true))
- throw new IllegalStateException("The GiraphGraphComputer job failed -- aborting all subsequent MapReduce jobs: " + job.getInternalJob().getStatus().getFailureInfo());
- // add vertex program memory values to the return memory
- for (final MemoryComputeKey memoryComputeKey : this.vertexProgram.getMemoryComputeKeys()) {
- if (!memoryComputeKey.isTransient() && storage.exists(Constants.getMemoryLocation(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION), memoryComputeKey.getKey()))) {
- final ObjectWritableIterator iterator = new ObjectWritableIterator(this.giraphConfiguration, new Path(Constants.getMemoryLocation(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION), memoryComputeKey.getKey())));
- if (iterator.hasNext()) {
- this.memory.set(memoryComputeKey.getKey(), iterator.next().getValue());
- }
- // vertex program memory items are not stored on disk
- storage.rm(Constants.getMemoryLocation(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION), memoryComputeKey.getKey()));
- }
- }
- final Path path = new Path(Constants.getMemoryLocation(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION), Constants.HIDDEN_ITERATION));
- this.memory.setIteration((Integer) new ObjectWritableIterator(this.giraphConfiguration, path).next().getValue());
- storage.rm(Constants.getMemoryLocation(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION), Constants.HIDDEN_ITERATION));
- }
- // do map reduce jobs
- this.giraphConfiguration.setBoolean(Constants.GREMLIN_HADOOP_GRAPH_READER_HAS_EDGES, this.giraphConfiguration.getBoolean(Constants.GREMLIN_HADOOP_GRAPH_WRITER_HAS_EDGES, true));
- for (final MapReduce mapReduce : this.mapReducers) {
- this.memory.addMapReduceMemoryKey(mapReduce);
- MapReduceHelper.executeMapReduceJob(mapReduce, this.memory, this.giraphConfiguration);
- }
-
- // if no persistence, delete the graph and memory output
- if (this.persist.equals(Persist.NOTHING))
- storage.rm(this.giraphConfiguration.get(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION));
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- return 0;
- }
-
- @Override
- public void setConf(final org.apache.hadoop.conf.Configuration configuration) {
- // TODO: is this necessary to implement?
- }
-
- @Override
- public org.apache.hadoop.conf.Configuration getConf() {
- return this.giraphConfiguration;
- }
-
- @Override
- protected void loadJar(final org.apache.hadoop.conf.Configuration hadoopConfiguration, final File file, final Object... params)
- throws IOException {
- final FileSystem defaultFileSystem = FileSystem.get(hadoopConfiguration);
- try {
- final Path jarFile = new Path(defaultFileSystem.getHomeDirectory() + "/hadoop-gremlin-" + Gremlin.version() + "-libs/" + file.getName());
- if (!defaultFileSystem.exists(jarFile)) {
- final Path sourcePath = new Path(file.getPath());
- final URI sourceUri = sourcePath.toUri();
- final FileSystem fs = FileSystem.get(sourceUri, hadoopConfiguration);
- fs.copyFromLocalFile(sourcePath, jarFile);
- }
- try {
- DistributedCache.addArchiveToClassPath(jarFile, this.giraphConfiguration, defaultFileSystem);
- } catch (final Exception e) {
- throw new RuntimeException(e.getMessage(), e);
- }
- } catch (final Exception e) {
- throw new IllegalStateException(e.getMessage(), e);
- }
- }
-
- public static void main(final String[] args) throws Exception {
- final FileConfiguration configuration = new PropertiesConfiguration(args[0]);
- new GiraphGraphComputer(HadoopGraph.open(configuration)).program(VertexProgram.createVertexProgram(HadoopGraph.open(configuration), configuration)).submit().get();
- }
-
- public Features features() {
- return new Features();
- }
-
- public class Features extends AbstractHadoopGraphComputer.Features {
-
- @Override
- public int getMaxWorkers() {
- if (GiraphGraphComputer.this.giraphConfiguration.getLocalTestMode())
- return Runtime.getRuntime().availableProcessors();
- else {
- return Integer.MAX_VALUE;
- /*try {
- final Cluster cluster = new Cluster(GiraphGraphComputer.this.giraphConfiguration);
- int maxWorkers = (cluster.getClusterStatus().getMapSlotCapacity() - 1) * 16; // max 16 threads per machine hardcoded :|
- cluster.close();
- return maxWorkers;
-
- } catch (final IOException | InterruptedException e) {
- throw new IllegalStateException(e.getMessage(), e);
- }*/
- }
- }
- }
-}