You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/05/05 21:39:55 UTC
[1/5] incubator-tinkerpop git commit: Make profiler for driver a bit
more robust.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master fa2150bbe -> 2c8139feb
Make profiler for driver a bit more robust.
Include a warmup session. Restructure for better re-use and setup for taking application arguments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f1768701
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f1768701
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f1768701
Branch: refs/heads/master
Commit: f176870193300ffcbd9cd4f52c3a34631648b9ce
Parents: 7a1a3aa
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 5 15:06:37 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 5 15:06:37 2015 -0400
----------------------------------------------------------------------
.../driver/benchmark/ProfilingApplication.java | 159 +++++++++++--------
1 file changed, 95 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f1768701/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
index 6b96eed..1f11ac0 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
@@ -21,7 +21,10 @@ package org.apache.tinkerpop.gremlin.driver.benchmark;
import org.apache.tinkerpop.gremlin.driver.Client;
import org.apache.tinkerpop.gremlin.driver.Cluster;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
@@ -34,12 +37,96 @@ import java.util.stream.IntStream;
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
public class ProfilingApplication {
+
+ private final Cluster cluster;
+ private final int requests;
+ private final int clients;
+
+ public ProfilingApplication(final Cluster cluster, final int clients, final int requests) {
+ this.cluster = cluster;
+ this.clients = clients;
+ this.requests = requests;
+ }
+
+ public long execute() throws Exception {
+ final Map<Thread, Long> rps = new ConcurrentHashMap<>();
+ final AtomicInteger tooSlow = new AtomicInteger(0);
+
+ // let all the clients fully init before starting to send messages
+ final CyclicBarrier barrier = new CyclicBarrier(clients);
+
+ final List<Thread> threads = IntStream.range(0, clients).mapToObj(t -> new Thread(() -> {
+ final Client client = cluster.connect();
+ try {
+ final CountDownLatch latch = new CountDownLatch(requests);
+ client.init();
+ barrier.await();
+
+ // timer starts after init of all clients
+ final long start = System.nanoTime();
+ IntStream.range(0, requests).forEach(i -> {
+ client.submitAsync("1+1").thenAcceptAsync(r -> {
+ try {
+ r.all().get(125, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException ex) {
+ tooSlow.incrementAndGet();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ } finally {
+ latch.countDown();
+ }
+ });
+ });
+
+ latch.await();
+
+ final long end = System.nanoTime();
+ final long total = end - start;
+
+ System.out.println("All responses for [" + t + "] are accounted for at: " + end);
+
+ final long totalSeconds = Math.round(total / 1000000000d);
+ final long requestCount = requests;
+ final long reqSec = Math.round(requestCount / totalSeconds);
+ rps.put(Thread.currentThread(), reqSec);
+ System.out.println(String.format("[" + t + "] clients: %s | requests: %s | time(s): %s | req/sec: %s | too slow: %s", clients, requestCount, totalSeconds, reqSec, tooSlow.get()));
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ } finally {
+ if (client != null) client.close();
+ }
+ }, "benchmark-client-" + t)).collect(Collectors.toList());
+
+ threads.forEach(t -> {
+ try {
+ t.start();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ });
+
+ threads.forEach(t -> {
+ try {
+ t.join();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ throw new RuntimeException(ex);
+ }
+ });
+
+ return rps.values().stream().collect(Collectors.averagingLong(l -> l)).longValue();
+ }
+
public static void main(final String[] args) {
try {
System.out.println("Initializing at: " + System.nanoTime());
final String host = args.length == 0 ? "localhost" : args[0];
+ final int warmups = 1;
+ final int executions = 1;
final int clients = 1;
final int requests = 10000;
final Cluster cluster = Cluster.build(host)
@@ -48,72 +135,16 @@ public class ProfilingApplication {
.nioPoolSize(clients)
.workerPoolSize(clients * 2).create();
- final AtomicInteger tooSlow = new AtomicInteger(0);
-
- // let all the clients fully init before starting to send messages
- final CyclicBarrier barrier = new CyclicBarrier(clients);
-
- final List<Thread> threads = IntStream.range(0, clients).mapToObj(t -> new Thread(() -> {
- try {
- final CountDownLatch latch = new CountDownLatch(requests);
-
- final Client client = cluster.connect();
- client.init();
-
- barrier.await();
- final long start = System.nanoTime();
-
- System.out.println("Executing at [" + t + "]:" + start);
-
- IntStream.range(0, requests).forEach(i -> {
- client.submitAsync("1+1").thenAcceptAsync(r -> {
- try {
- r.all().get(100, TimeUnit.MILLISECONDS);
- } catch (TimeoutException ex) {
- tooSlow.incrementAndGet();
- } catch (Exception ex) {
- ex.printStackTrace();
- } finally {
- latch.countDown();
- }
- });
- });
+ for (int ix = 0; ix < warmups; ix ++) {
+ new ProfilingApplication(cluster, clients, requests).execute();
+ }
- latch.await();
-
- final long end = System.nanoTime();
- final long total = end - start;
-
- System.out.println("All responses for [" + t + "] are accounted for at: " + end);
-
- final long totalSeconds = Math.round(total / 1000000000d);
- final long requestCount = requests;
- final long reqSec = Math.round(requestCount / totalSeconds);
- System.out.println(String.format("[" + t + "] clients: %s | requests: %s | time(s): %s | req/sec: %s | too slow: %s", clients, requestCount, totalSeconds, reqSec, tooSlow.get()));
- } catch (Exception ex) {
- ex.printStackTrace();
- throw new RuntimeException(ex);
- }
- })).collect(Collectors.toList());
-
- threads.forEach(t -> {
- try {
- t.start();
- } catch (Exception ex) {
- ex.printStackTrace();
- throw new RuntimeException(ex);
- }
- });
-
- threads.forEach(t -> {
- try {
- t.join();
- } catch (Exception ex) {
- ex.printStackTrace();
- throw new RuntimeException(ex);
- }
- });
+ long totalRequestsPerSecond = 0;
+ for (int ix = 0; ix < executions; ix ++) {
+ totalRequestsPerSecond += new ProfilingApplication(cluster, clients, requests).execute();
+ }
+ System.out.println(String.format("avg req/sec: %s", totalRequestsPerSecond / executions));
} catch (Exception ex) {
ex.printStackTrace();
} finally {
[5/5] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master'
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2c8139fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2c8139fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2c8139fe
Branch: refs/heads/master
Commit: 2c8139febde2c1906514dadb2294d5494ca7bec9
Parents: 992fe85 fa2150b
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 5 15:39:44 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 5 15:39:44 2015 -0400
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/implementations.asciidoc | 129 ++++--------------
.../process/traversal/TraversalStrategies.java | 15 +--
.../traversal/lambda/TokenTraversal.java | 2 +-
.../AdjacentToIncidentStrategy.java | 82 ++++++++++++
.../AdjacentToIncidentStrategyTest.java | 130 +++++++++++++++++++
hadoop-gremlin/conf/hadoop-graphson.properties | 2 +-
hadoop-gremlin/conf/hadoop-gryo.properties | 2 +-
.../tinkergraph/structure/TinkerGraphTest.java | 21 +--
9 files changed, 247 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
[4/5] incubator-tinkerpop git commit: Minor adjustments to log output.
Posted by sp...@apache.org.
Minor adjustments to log output.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/992fe85d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/992fe85d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/992fe85d
Branch: refs/heads/master
Commit: 992fe85dda507a70d2422add2b7cff74701bba65
Parents: 14924f1
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 5 15:39:16 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 5 15:39:16 2015 -0400
----------------------------------------------------------------------
.../driver/benchmark/ProfilingApplication.java | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/992fe85d/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
index 4d6d111..675a11b 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
@@ -41,10 +41,10 @@ public class ProfilingApplication {
private final Cluster cluster;
private final int requests;
private final int clients;
- private final int execution;
+ private final String executionName;
- public ProfilingApplication(final int execution, final Cluster cluster, final int clients, final int requests) {
- this.execution = execution;
+ public ProfilingApplication(final String executionName, final Cluster cluster, final int clients, final int requests) {
+ this.executionName = executionName;
this.cluster = cluster;
this.clients = clients;
this.requests = requests;
@@ -59,7 +59,7 @@ public class ProfilingApplication {
final List<Thread> threads = IntStream.range(0, clients).mapToObj(t -> new Thread(() -> {
final Client client = cluster.connect();
- final String executionId = "[" + execution + "-" + t + "]";
+ final String executionId = "[" + executionName + "-" + (t + 1) + "]";
try {
final CountDownLatch latch = new CountDownLatch(requests);
client.init();
@@ -81,6 +81,7 @@ public class ProfilingApplication {
});
});
+ // finish once all requests are accounted for
latch.await();
final long end = System.nanoTime();
@@ -89,6 +90,7 @@ public class ProfilingApplication {
final long requestCount = requests;
final long reqSec = Math.round(requestCount / totalSeconds);
rps.put(Thread.currentThread(), reqSec);
+
System.out.println(String.format(executionId + " clients: %s | requests: %s | time(s): %s | req/sec: %s | too slow: %s", clients, requestCount, totalSeconds, reqSec, tooSlow.get()));
} catch (Exception ex) {
ex.printStackTrace();
@@ -96,7 +98,7 @@ public class ProfilingApplication {
} finally {
if (client != null) client.close();
}
- }, "benchmark-client-" + execution + "-" + t)).collect(Collectors.toList());
+ }, "benchmark-client-" + executionName + "-" + (t + 1))).collect(Collectors.toList());
threads.forEach(t -> {
try {
@@ -126,7 +128,7 @@ public class ProfilingApplication {
final int warmups = 3;
final int executions = 10;
final int clients = 1;
- final int requests = 10000;
+ final int requests = 100000;
final Cluster cluster = Cluster.build(host)
.minConnectionPoolSize(256)
.maxConnectionPoolSize(256)
@@ -134,12 +136,12 @@ public class ProfilingApplication {
.workerPoolSize(clients * 2).create();
for (int ix = 0; ix < warmups; ix ++) {
- new ProfilingApplication(ix + 1, cluster, clients, requests).execute();
+ new ProfilingApplication("warmup-" + (ix + 1), cluster, clients, requests).execute();
}
long totalRequestsPerSecond = 0;
for (int ix = 0; ix < executions; ix ++) {
- totalRequestsPerSecond += new ProfilingApplication(ix + 1, cluster, clients, requests).execute();
+ totalRequestsPerSecond += new ProfilingApplication("test-" + (ix + 1), cluster, clients, requests).execute();
}
System.out.println(String.format("avg req/sec: %s", totalRequestsPerSecond / executions));
[3/5] incubator-tinkerpop git commit: Minor adjustments to log output
and thread naming.
Posted by sp...@apache.org.
Minor adjustments to log output and thread naming.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/14924f17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/14924f17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/14924f17
Branch: refs/heads/master
Commit: 14924f17b64a2a68249999a7c8a998c77af6f2ed
Parents: 61c1ac6
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 5 15:27:28 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 5 15:27:28 2015 -0400
----------------------------------------------------------------------
.../gremlin/driver/benchmark/ProfilingApplication.java | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/14924f17/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
index e7178cf..4d6d111 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
@@ -41,8 +41,10 @@ public class ProfilingApplication {
private final Cluster cluster;
private final int requests;
private final int clients;
+ private final int execution;
- public ProfilingApplication(final Cluster cluster, final int clients, final int requests) {
+ public ProfilingApplication(final int execution, final Cluster cluster, final int clients, final int requests) {
+ this.execution = execution;
this.cluster = cluster;
this.clients = clients;
this.requests = requests;
@@ -57,6 +59,7 @@ public class ProfilingApplication {
final List<Thread> threads = IntStream.range(0, clients).mapToObj(t -> new Thread(() -> {
final Client client = cluster.connect();
+ final String executionId = "[" + execution + "-" + t + "]";
try {
final CountDownLatch latch = new CountDownLatch(requests);
client.init();
@@ -86,14 +89,14 @@ public class ProfilingApplication {
final long requestCount = requests;
final long reqSec = Math.round(requestCount / totalSeconds);
rps.put(Thread.currentThread(), reqSec);
- System.out.println(String.format("[" + t + "] clients: %s | requests: %s | time(s): %s | req/sec: %s | too slow: %s", clients, requestCount, totalSeconds, reqSec, tooSlow.get()));
+ System.out.println(String.format(executionId + " clients: %s | requests: %s | time(s): %s | req/sec: %s | too slow: %s", clients, requestCount, totalSeconds, reqSec, tooSlow.get()));
} catch (Exception ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
} finally {
if (client != null) client.close();
}
- }, "benchmark-client-" + t)).collect(Collectors.toList());
+ }, "benchmark-client-" + execution + "-" + t)).collect(Collectors.toList());
threads.forEach(t -> {
try {
@@ -131,12 +134,12 @@ public class ProfilingApplication {
.workerPoolSize(clients * 2).create();
for (int ix = 0; ix < warmups; ix ++) {
- new ProfilingApplication(cluster, clients, requests).execute();
+ new ProfilingApplication(ix + 1, cluster, clients, requests).execute();
}
long totalRequestsPerSecond = 0;
for (int ix = 0; ix < executions; ix ++) {
- totalRequestsPerSecond += new ProfilingApplication(cluster, clients, requests).execute();
+ totalRequestsPerSecond += new ProfilingApplication(ix + 1, cluster, clients, requests).execute();
}
System.out.println(String.format("avg req/sec: %s", totalRequestsPerSecond / executions));
[2/5] incubator-tinkerpop git commit: Remove some of the printlns
from the profiler.
Posted by sp...@apache.org.
Remove some of the printlns from the profiler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/61c1ac64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/61c1ac64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/61c1ac64
Branch: refs/heads/master
Commit: 61c1ac645aa9971bdafe3b2feb6517eeb4e4e669
Parents: f176870
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Tue May 5 15:19:21 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Tue May 5 15:19:21 2015 -0400
----------------------------------------------------------------------
.../gremlin/driver/benchmark/ProfilingApplication.java | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/61c1ac64/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
----------------------------------------------------------------------
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
index 1f11ac0..e7178cf 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/benchmark/ProfilingApplication.java
@@ -82,9 +82,6 @@ public class ProfilingApplication {
final long end = System.nanoTime();
final long total = end - start;
-
- System.out.println("All responses for [" + t + "] are accounted for at: " + end);
-
final long totalSeconds = Math.round(total / 1000000000d);
final long requestCount = requests;
final long reqSec = Math.round(requestCount / totalSeconds);
@@ -121,12 +118,10 @@ public class ProfilingApplication {
public static void main(final String[] args) {
try {
- System.out.println("Initializing at: " + System.nanoTime());
-
final String host = args.length == 0 ? "localhost" : args[0];
- final int warmups = 1;
- final int executions = 1;
+ final int warmups = 3;
+ final int executions = 10;
final int clients = 1;
final int requests = 10000;
final Cluster cluster = Cluster.build(host)