You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/18 23:06:35 UTC
incubator-tinkerpop git commit: Added MapReduce pool which is
analagous to VertexProgramPool -- and is smart about StaticMapReduce. Rewrote
the TinkerWorkerPool for TinkerGraphComputer to use VertexProgramPool and
MapReducePool. Less object creation and
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 560ed9b9c -> 4155961ae
Added MapReduce pool which is analagous to VertexProgramPool -- and is smart about StaticMapReduce. Rewrote the TinkerWorkerPool for TinkerGraphComputer to use VertexProgramPool and MapReducePool. Less object creation and start/end is not threaded.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4155961a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4155961a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4155961a
Branch: refs/heads/master
Commit: 4155961ae9ce491d42e42dec89183f5f075ec723
Parents: 560ed9b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Mar 18 16:06:32 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Mar 18 16:06:32 2015 -0600
----------------------------------------------------------------------
.../process/computer/util/MapReducePool.java | 88 ++++++++++++++++++++
.../computer/util/VertexProgramPool.java | 5 +-
.../computer/giraph/GiraphWorkerContext.java | 7 +-
.../process/computer/TinkerGraphComputer.java | 14 ++--
.../process/computer/TinkerWorkerPool.java | 71 +++++++++-------
5 files changed, 139 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java
new file mode 100644
index 0000000..167b01a
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.util;
+
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class MapReducePool {
+ private final LinkedBlockingQueue<MapReduce<?, ?, ?, ?, ?>> pool;
+ private final StaticMapReduce mapReduce;
+ private static final int TIMEOUT_MS = 2500;
+
+ public MapReducePool(final MapReduce mapReduce, final int poolSize) {
+ if (mapReduce instanceof StaticMapReduce) {
+ this.pool = null;
+ this.mapReduce = (StaticMapReduce) mapReduce;
+ } else {
+ this.mapReduce = null;
+ this.pool = new LinkedBlockingQueue<>(poolSize);
+ while (this.pool.remainingCapacity() > 0) {
+ this.pool.add(mapReduce.clone());
+ }
+ }
+ }
+
+ public MapReduce take() {
+ if (null == this.mapReduce) {
+ try {
+ return this.pool.poll(TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ } catch (final InterruptedException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ } else {
+ return this.mapReduce;
+ }
+ }
+
+ public void offer(final MapReduce<?, ?, ?, ?, ?> mapReduce) {
+ if (null == this.mapReduce) {
+ try {
+ this.pool.offer(mapReduce, TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ } catch (final InterruptedException e) {
+ throw new IllegalStateException(e.getMessage(), e);
+ }
+ }
+ }
+
+ /*public synchronized void workerIterationStart(final Memory memory) {
+ if (null == this.mapReduce) {
+ for (final MapReduce<?,?,?,?,?> mapReduce : this.pool) {
+ mapReduce.workerIterationStart(memory);
+ }
+ } else {
+ this.mapReduce.workerIterationStart(memory);
+ }
+ }*/
+
+ /*public synchronized void workerIterationEnd(final Memory memory) {
+ if (null == this.mapReduce) {
+ for (final MapReduce<?,?,?,?,?> mapReduce : this.pool) {
+ mapReduce.workerIterationEnd(memory);
+ }
+ } else {
+ this.mapReduce.workerIterationEnd(memory);
+ }
+ }*/
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
index f3cb529..509b16d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
@@ -18,7 +18,6 @@
*/
package org.apache.tinkerpop.gremlin.process.computer.util;
-import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
@@ -34,8 +33,7 @@ public final class VertexProgramPool {
private final StaticVertexProgram vertexProgram;
private static final int TIMEOUT_MS = 2500;
- public VertexProgramPool(final int poolSize, final Configuration configuration) {
- final VertexProgram vertexProgram = VertexProgram.createVertexProgram(configuration);
+ public VertexProgramPool(final VertexProgram vertexProgram, final int poolSize) {
if (vertexProgram instanceof StaticVertexProgram) {
this.pool = null;
this.vertexProgram = (StaticVertexProgram) vertexProgram;
@@ -89,5 +87,4 @@ public final class VertexProgramPool {
this.vertexProgram.workerIterationEnd(memory);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
index 2017a6b..dc4d51a 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
@@ -44,10 +44,9 @@ public final class GiraphWorkerContext extends WorkerContext {
}
public void preApplication() throws InstantiationException, IllegalAccessException {
- this.vertexProgramPool = new VertexProgramPool(
- this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1),
- ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration()));
- this.memory = new GiraphMemory(this, VertexProgram.createVertexProgram(ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration())));
+ final VertexProgram vertexProgram = VertexProgram.createVertexProgram(ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration()));
+ this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
+ this.memory = new GiraphMemory(this, vertexProgram);
this.deriveMemory = this.getContext().getConfiguration().getBoolean(Constants.GREMLIN_HADOOP_DERIVE_MEMORY, false);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
index 3f05dbe..6e55987 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
@@ -94,7 +94,7 @@ public class TinkerGraphComputer implements GraphComputer {
GraphComputerHelper.validateProgramOnComputer(this, this.vertexProgram);
this.mapReducers.addAll(this.vertexProgram.getMapReducers());
}
- //final Graph computeGraph = this.graph;
+ //final Graph computeGraph = this.graph;
final Graph computeGraph = null == this.vertexProgram ? this.graph :
new ComputerGraph(this.graph, this.vertexProgram.getElementComputeKeys());
@@ -109,7 +109,8 @@ public class TinkerGraphComputer implements GraphComputer {
this.vertexProgram.setup(this.memory);
this.memory.completeSubRound();
while (true) {
- workers.executeVertexProgram(vertexProgram -> vertexProgram.workerIterationStart(this.memory.asImmutable()), vertexProgram);
+ workers.setVertexProgram(this.vertexProgram);
+ workers.vertexProgramWorkerIterationStart(this.memory.asImmutable());
final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(computeGraph.vertices());
workers.executeVertexProgram(vertexProgram -> {
while (true) {
@@ -117,8 +118,8 @@ public class TinkerGraphComputer implements GraphComputer {
if (null == vertex) return;
vertexProgram.execute(vertex, new TinkerMessenger(vertex, this.messageBoard, vertexProgram.getMessageCombiner()), this.memory);
}
- }, vertexProgram);
- workers.executeVertexProgram(vertexProgram -> vertexProgram.workerIterationEnd(this.memory.asImmutable()), vertexProgram);
+ });
+ workers.vertexProgramWorkerIterationEnd(this.memory.asImmutable());
this.messageBoard.completeIteration();
this.memory.completeSubRound();
if (this.vertexProgram.terminate(this.memory)) {
@@ -137,13 +138,14 @@ public class TinkerGraphComputer implements GraphComputer {
if (mapReduce.doStage(MapReduce.Stage.MAP)) {
final TinkerMapEmitter<?, ?> mapEmitter = new TinkerMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(computeGraph.vertices());
+ workers.setMapReduce(mapReduce);
workers.executeMapReduce(workerMapReduce -> {
while (true) {
final Vertex vertex = vertices.next();
if (null == vertex) return;
workerMapReduce.map(vertex, mapEmitter);
}
- }, mapReduce);
+ });
// sort results if a map output sort is defined
mapEmitter.complete(mapReduce);
@@ -158,7 +160,7 @@ public class TinkerGraphComputer implements GraphComputer {
if (null == entry) return;
workerMapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter);
}
- }, mapReduce);
+ });
reduceEmitter.complete(mapReduce); // sort results if a reduce output sort is defined
mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator());
} else {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
index 2625b5a..2e0030d 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
@@ -20,16 +20,14 @@ package org.apache.tinkerpop.gremlin.tinkergraph.process.computer;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.util.function.FunctionUtils;
+import org.apache.tinkerpop.gremlin.process.computer.util.MapReducePool;
+import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.function.Consumer;
-import java.util.stream.Collectors;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -42,47 +40,56 @@ public class TinkerWorkerPool implements AutoCloseable {
private final int numberOfWorkers;
private final ExecutorService workerPool;
+ private VertexProgramPool vertexProgramPool;
+ private MapReducePool mapReducePool;
+
public TinkerWorkerPool(final int numberOfWorkers) {
this.numberOfWorkers = numberOfWorkers;
workerPool = Executors.newFixedThreadPool(numberOfWorkers, threadFactoryWorker);
}
- public void executeVertexProgram(final Consumer<VertexProgram> worker, final VertexProgram vertexProgram) {
- final List<VertexProgram> vertexPrograms = cloneVertexProgram(vertexProgram);
- vertexPrograms.stream()
- .map(vp -> (Future<Void>) workerPool.submit(() -> worker.accept(vp)))
- .collect(Collectors.toList())
- .forEach(FunctionUtils.wrapConsumer(Future::get));
+ public void setVertexProgram(final VertexProgram vertexProgram) {
+ this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.numberOfWorkers);
}
- public void executeMapReduce(final Consumer<MapReduce> worker, final MapReduce mapReduce) {
- final List<MapReduce> mapReducers = cloneMapReducer(mapReduce);
- mapReducers.stream()
- .map(mr -> (Future<Void>) workerPool.submit(() -> worker.accept(mr)))
- .collect(Collectors.toList())
- .forEach(FunctionUtils.wrapConsumer(Future::get));
+ public void setMapReduce(final MapReduce mapReduce) {
+ this.mapReducePool = new MapReducePool(mapReduce, this.numberOfWorkers);
}
- @Override
- public void close() throws Exception {
- workerPool.shutdown();
+ public void vertexProgramWorkerIterationStart(final Memory memory) {
+ this.vertexProgramPool.workerIterationStart(memory);
+ }
+
+ public void vertexProgramWorkerIterationEnd(final Memory memory) {
+ this.vertexProgramPool.workerIterationEnd(memory);
}
- private List<VertexProgram> cloneVertexProgram(final VertexProgram vertexProgram) {
- final List<VertexProgram> vertexPrograms;
- vertexPrograms = new ArrayList<>(numberOfWorkers);
- for (int i = 0; i < numberOfWorkers; i++) {
- vertexPrograms.add(vertexProgram.clone());
+ public void executeVertexProgram(final Consumer<VertexProgram> worker) {
+ try {
+ this.workerPool.submit(() -> {
+ final VertexProgram vp = this.vertexProgramPool.take();
+ worker.accept(vp);
+ this.vertexProgramPool.offer(vp);
+ }).get();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
}
- return vertexPrograms;
}
- private List<MapReduce> cloneMapReducer(final MapReduce mapReduce) {
- final List<MapReduce> mapReducers;
- mapReducers = new ArrayList<>(numberOfWorkers);
- for (int i = 0; i < numberOfWorkers; i++) {
- mapReducers.add(mapReduce.clone());
+ public void executeMapReduce(final Consumer<MapReduce> worker) {
+ try {
+ this.workerPool.submit(() -> {
+ final MapReduce mr = this.mapReducePool.take();
+ worker.accept(mr);
+ this.mapReducePool.offer(mr);
+ }).get();
+ } catch (final Exception e) {
+ throw new IllegalStateException(e.getMessage(), e);
}
- return mapReducers;
+ }
+
+ @Override
+ public void close() throws Exception {
+ workerPool.shutdown();
}
}
\ No newline at end of file