You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/03/18 23:06:35 UTC

incubator-tinkerpop git commit: Added MapReduce pool which is analagous to VertexProgramPool -- and is smart about StaticMapReduce. Rewrote the TinkerWorkerPool for TinkerGraphComputer to use VertexProgramPool and MapReducePool. Less object creation and

Repository: incubator-tinkerpop
Updated Branches:
  refs/heads/master 560ed9b9c -> 4155961ae


Added MapReduce pool which is analagous to VertexProgramPool -- and is smart about StaticMapReduce. Rewrote the TinkerWorkerPool for TinkerGraphComputer to use VertexProgramPool and MapReducePool. Less object creation and start/end is not threaded.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/4155961a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/4155961a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/4155961a

Branch: refs/heads/master
Commit: 4155961ae9ce491d42e42dec89183f5f075ec723
Parents: 560ed9b
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Wed Mar 18 16:06:32 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Wed Mar 18 16:06:32 2015 -0600

----------------------------------------------------------------------
 .../process/computer/util/MapReducePool.java    | 88 ++++++++++++++++++++
 .../computer/util/VertexProgramPool.java        |  5 +-
 .../computer/giraph/GiraphWorkerContext.java    |  7 +-
 .../process/computer/TinkerGraphComputer.java   | 14 ++--
 .../process/computer/TinkerWorkerPool.java      | 71 +++++++++-------
 5 files changed, 139 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java
new file mode 100644
index 0000000..167b01a
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/MapReducePool.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.util;
+
+import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public final class MapReducePool {
+    private final LinkedBlockingQueue<MapReduce<?, ?, ?, ?, ?>> pool;
+    private final StaticMapReduce mapReduce;
+    private static final int TIMEOUT_MS = 2500;
+
+    public MapReducePool(final MapReduce mapReduce, final int poolSize) {
+        if (mapReduce instanceof StaticMapReduce) {
+            this.pool = null;
+            this.mapReduce = (StaticMapReduce) mapReduce;
+        } else {
+            this.mapReduce = null;
+            this.pool = new LinkedBlockingQueue<>(poolSize);
+            while (this.pool.remainingCapacity() > 0) {
+                this.pool.add(mapReduce.clone());
+            }
+        }
+    }
+
+    public MapReduce take() {
+        if (null == this.mapReduce) {
+            try {
+                return this.pool.poll(TIMEOUT_MS, TimeUnit.MILLISECONDS);
+            } catch (final InterruptedException e) {
+                throw new IllegalStateException(e.getMessage(), e);
+            }
+        } else {
+            return this.mapReduce;
+        }
+    }
+
+    public void offer(final MapReduce<?, ?, ?, ?, ?> mapReduce) {
+        if (null == this.mapReduce) {
+            try {
+                this.pool.offer(mapReduce, TIMEOUT_MS, TimeUnit.MILLISECONDS);
+            } catch (final InterruptedException e) {
+                throw new IllegalStateException(e.getMessage(), e);
+            }
+        }
+    }
+
+    /*public synchronized void workerIterationStart(final Memory memory) {
+        if (null == this.mapReduce) {
+            for (final MapReduce<?,?,?,?,?> mapReduce : this.pool) {
+                mapReduce.workerIterationStart(memory);
+            }
+        } else {
+            this.mapReduce.workerIterationStart(memory);
+        }
+    }*/
+
+    /*public synchronized void workerIterationEnd(final Memory memory) {
+        if (null == this.mapReduce) {
+            for (final MapReduce<?,?,?,?,?> mapReduce : this.pool) {
+                mapReduce.workerIterationEnd(memory);
+            }
+        } else {
+            this.mapReduce.workerIterationEnd(memory);
+        }
+    }*/
+}

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
index f3cb529..509b16d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/util/VertexProgramPool.java
@@ -18,7 +18,6 @@
  */
 package org.apache.tinkerpop.gremlin.process.computer.util;
 
-import org.apache.commons.configuration.Configuration;
 import org.apache.tinkerpop.gremlin.process.computer.Memory;
 import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
 
@@ -34,8 +33,7 @@ public final class VertexProgramPool {
     private final StaticVertexProgram vertexProgram;
     private static final int TIMEOUT_MS = 2500;
 
-    public VertexProgramPool(final int poolSize, final Configuration configuration) {
-        final VertexProgram vertexProgram = VertexProgram.createVertexProgram(configuration);
+    public VertexProgramPool(final VertexProgram vertexProgram, final int poolSize) {
         if (vertexProgram instanceof StaticVertexProgram) {
             this.pool = null;
             this.vertexProgram = (StaticVertexProgram) vertexProgram;
@@ -89,5 +87,4 @@ public final class VertexProgramPool {
             this.vertexProgram.workerIterationEnd(memory);
         }
     }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
index 2017a6b..dc4d51a 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
@@ -44,10 +44,9 @@ public final class GiraphWorkerContext extends WorkerContext {
     }
 
     public void preApplication() throws InstantiationException, IllegalAccessException {
-        this.vertexProgramPool = new VertexProgramPool(
-                this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1),
-                ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration()));
-        this.memory = new GiraphMemory(this, VertexProgram.createVertexProgram(ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration())));
+        final VertexProgram vertexProgram = VertexProgram.createVertexProgram(ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration()));
+        this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
+        this.memory = new GiraphMemory(this, vertexProgram);
         this.deriveMemory = this.getContext().getConfiguration().getBoolean(Constants.GREMLIN_HADOOP_DERIVE_MEMORY, false);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
index 3f05dbe..6e55987 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
@@ -94,7 +94,7 @@ public class TinkerGraphComputer implements GraphComputer {
             GraphComputerHelper.validateProgramOnComputer(this, this.vertexProgram);
             this.mapReducers.addAll(this.vertexProgram.getMapReducers());
         }
-          //final Graph computeGraph = this.graph;
+        //final Graph computeGraph = this.graph;
         final Graph computeGraph = null == this.vertexProgram ? this.graph :
                 new ComputerGraph(this.graph, this.vertexProgram.getElementComputeKeys());
 
@@ -109,7 +109,8 @@ public class TinkerGraphComputer implements GraphComputer {
                     this.vertexProgram.setup(this.memory);
                     this.memory.completeSubRound();
                     while (true) {
-                        workers.executeVertexProgram(vertexProgram -> vertexProgram.workerIterationStart(this.memory.asImmutable()), vertexProgram);
+                        workers.setVertexProgram(this.vertexProgram);
+                        workers.vertexProgramWorkerIterationStart(this.memory.asImmutable());
                         final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(computeGraph.vertices());
                         workers.executeVertexProgram(vertexProgram -> {
                             while (true) {
@@ -117,8 +118,8 @@ public class TinkerGraphComputer implements GraphComputer {
                                 if (null == vertex) return;
                                 vertexProgram.execute(vertex, new TinkerMessenger(vertex, this.messageBoard, vertexProgram.getMessageCombiner()), this.memory);
                             }
-                        }, vertexProgram);
-                        workers.executeVertexProgram(vertexProgram -> vertexProgram.workerIterationEnd(this.memory.asImmutable()), vertexProgram);
+                        });
+                        workers.vertexProgramWorkerIterationEnd(this.memory.asImmutable());
                         this.messageBoard.completeIteration();
                         this.memory.completeSubRound();
                         if (this.vertexProgram.terminate(this.memory)) {
@@ -137,13 +138,14 @@ public class TinkerGraphComputer implements GraphComputer {
                     if (mapReduce.doStage(MapReduce.Stage.MAP)) {
                         final TinkerMapEmitter<?, ?> mapEmitter = new TinkerMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
                         final SynchronizedIterator<Vertex> vertices = new SynchronizedIterator<>(computeGraph.vertices());
+                        workers.setMapReduce(mapReduce);
                         workers.executeMapReduce(workerMapReduce -> {
                             while (true) {
                                 final Vertex vertex = vertices.next();
                                 if (null == vertex) return;
                                 workerMapReduce.map(vertex, mapEmitter);
                             }
-                        }, mapReduce);
+                        });
 
                         // sort results if a map output sort is defined
                         mapEmitter.complete(mapReduce);
@@ -158,7 +160,7 @@ public class TinkerGraphComputer implements GraphComputer {
                                     if (null == entry) return;
                                     workerMapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter);
                                 }
-                            }, mapReduce);
+                            });
                             reduceEmitter.complete(mapReduce); // sort results if a reduce output sort is defined
                             mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator());
                         } else {

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/4155961a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
index 2625b5a..2e0030d 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerWorkerPool.java
@@ -20,16 +20,14 @@ package org.apache.tinkerpop.gremlin.tinkergraph.process.computer;
 
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
 import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
-import org.apache.tinkerpop.gremlin.util.function.FunctionUtils;
+import org.apache.tinkerpop.gremlin.process.computer.util.MapReducePool;
+import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramPool;
 
-import java.util.ArrayList;
-import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
 import java.util.function.Consumer;
-import java.util.stream.Collectors;
 
 /**
  * @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -42,47 +40,56 @@ public class TinkerWorkerPool implements AutoCloseable {
     private final int numberOfWorkers;
     private final ExecutorService workerPool;
 
+    private VertexProgramPool vertexProgramPool;
+    private MapReducePool mapReducePool;
+
     public TinkerWorkerPool(final int numberOfWorkers) {
         this.numberOfWorkers = numberOfWorkers;
         workerPool = Executors.newFixedThreadPool(numberOfWorkers, threadFactoryWorker);
     }
 
-    public void executeVertexProgram(final Consumer<VertexProgram> worker, final VertexProgram vertexProgram) {
-        final List<VertexProgram> vertexPrograms = cloneVertexProgram(vertexProgram);
-        vertexPrograms.stream()
-                .map(vp -> (Future<Void>) workerPool.submit(() -> worker.accept(vp)))
-                .collect(Collectors.toList())
-                .forEach(FunctionUtils.wrapConsumer(Future::get));
+    public void setVertexProgram(final VertexProgram vertexProgram) {
+        this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.numberOfWorkers);
     }
 
-    public void executeMapReduce(final Consumer<MapReduce> worker, final MapReduce mapReduce) {
-        final List<MapReduce> mapReducers = cloneMapReducer(mapReduce);
-        mapReducers.stream()
-                .map(mr -> (Future<Void>) workerPool.submit(() -> worker.accept(mr)))
-                .collect(Collectors.toList())
-                .forEach(FunctionUtils.wrapConsumer(Future::get));
+    public void setMapReduce(final MapReduce mapReduce) {
+        this.mapReducePool = new MapReducePool(mapReduce, this.numberOfWorkers);
     }
 
-    @Override
-    public void close() throws Exception {
-        workerPool.shutdown();
+    public void vertexProgramWorkerIterationStart(final Memory memory) {
+        this.vertexProgramPool.workerIterationStart(memory);
+    }
+
+    public void vertexProgramWorkerIterationEnd(final Memory memory) {
+        this.vertexProgramPool.workerIterationEnd(memory);
     }
 
-    private List<VertexProgram> cloneVertexProgram(final VertexProgram vertexProgram) {
-        final List<VertexProgram> vertexPrograms;
-        vertexPrograms = new ArrayList<>(numberOfWorkers);
-        for (int i = 0; i < numberOfWorkers; i++) {
-            vertexPrograms.add(vertexProgram.clone());
+    public void executeVertexProgram(final Consumer<VertexProgram> worker) {
+        try {
+            this.workerPool.submit(() -> {
+                final VertexProgram vp = this.vertexProgramPool.take();
+                worker.accept(vp);
+                this.vertexProgramPool.offer(vp);
+            }).get();
+        } catch (final Exception e) {
+            throw new IllegalStateException(e.getMessage(), e);
         }
-        return vertexPrograms;
     }
 
-    private List<MapReduce> cloneMapReducer(final MapReduce mapReduce) {
-        final List<MapReduce> mapReducers;
-        mapReducers = new ArrayList<>(numberOfWorkers);
-        for (int i = 0; i < numberOfWorkers; i++) {
-            mapReducers.add(mapReduce.clone());
+    public void executeMapReduce(final Consumer<MapReduce> worker) {
+        try {
+            this.workerPool.submit(() -> {
+                final MapReduce mr = this.mapReducePool.take();
+                worker.accept(mr);
+                this.mapReducePool.offer(mr);
+            }).get();
+        } catch (final Exception e) {
+            throw new IllegalStateException(e.getMessage(), e);
         }
-        return mapReducers;
+    }
+
+    @Override
+    public void close() throws Exception {
+        workerPool.shutdown();
     }
 }
\ No newline at end of file