You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/05/29 07:18:38 UTC
[22/30] ignite git commit: ignite-5075
ignite-5075
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9585dd32
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9585dd32
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9585dd32
Branch: refs/heads/ignite-5075-pds
Commit: 9585dd326c9cdc08a6c1e522e663939d516b6e1d
Parents: 6a8a832
Author: sboikov <sb...@gridgain.com>
Authored: Fri May 26 14:45:01 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri May 26 14:45:01 2017 +0300
----------------------------------------------------------------------
.../cache/IgniteCacheAbstractBenchmark.java | 58 ++++++++++++++++++++
.../yardstick/cache/IgniteGetBenchmark.java | 47 +---------------
.../cache/IgniteSqlQueryBenchmark.java | 14 +++--
.../IgniteSqlQueryDistributedJoinBenchmark.java | 24 ++++----
.../cache/IgniteSqlQueryJoinBenchmark.java | 14 +++--
.../cache/IgniteSqlQueryPutBenchmark.java | 8 +--
.../IgniteSqlQueryPutSeparatedBenchmark.java | 4 ++
7 files changed, 99 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9585dd32/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
index 9d0b01b..183f478 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteCacheAbstractBenchmark.java
@@ -19,9 +19,13 @@ package org.apache.ignite.yardstick.cache;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.IgniteCache;
@@ -36,6 +40,8 @@ import org.apache.ignite.yardstick.IgniteNode;
import org.yardstickframework.BenchmarkConfiguration;
import org.yardstickframework.BenchmarkUtils;
+import static org.yardstickframework.BenchmarkUtils.println;
+
/**
* Abstract class for Ignite benchmarks which use cache.
*/
@@ -159,6 +165,58 @@ public abstract class IgniteCacheAbstractBenchmark<K, V> extends IgniteAbstractB
}
/**
+ * @throws Exception If failed.
+ */
+ protected final void loadCachesData() throws Exception {
+ List<IgniteCache> caches = grpCaches != null ? grpCaches : (List)Collections.singletonList(cache);
+
+ if (caches.size() > 1) {
+ ExecutorService executor = Executors.newFixedThreadPool(10);
+
+ try {
+ List<Future<?>> futs = new ArrayList<>();
+
+ for (final IgniteCache cache : caches) {
+ futs.add(executor.submit(new Runnable() {
+ @Override public void run() {
+ loadCacheData0(cache.getName());
+ }
+ }));
+ }
+
+ for (Future<?> fut : futs)
+ fut.get();
+ }
+ finally {
+ executor.shutdown();
+ }
+ }
+ else
+ loadCacheData(caches.get(0).getName());
+ }
+
+ /**
+ * @param cacheName Cache name.
+ */
+ private void loadCacheData0(String cacheName) {
+ println(cfg, "Loading data for cache: " + cacheName);
+
+ long start = System.nanoTime();
+
+ loadCacheData(cacheName);
+
+ println(cfg, "Finished populating data [cache=" + cacheName +
+ ", time=" + ((System.nanoTime() - start) / 1_000_000) + "ms]");
+ }
+
+ /**
+ * @param cacheName Cache name.
+ */
+ protected void loadCacheData(String cacheName) {
+ throw new IllegalStateException("Not implemented for " + getClass().getSimpleName());
+ }
+
+ /**
* @return Range.
*/
protected final ThreadRange threadRange() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/9585dd32/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
index 9d1e386..8beb043 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteGetBenchmark.java
@@ -17,13 +17,7 @@
package org.apache.ignite.yardstick.cache;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.yardstick.cache.model.SampleValue;
@@ -43,43 +37,11 @@ public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Ob
throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") " +
"must by less then the range (\"-r\", \"--range\").");
- if (args.preloadAmount() > 0) {
- List<IgniteCache> caches = grpCaches != null ? grpCaches : (List)Collections.singletonList(cache);
-
- if (caches.size() > 1) {
- ExecutorService executor = Executors.newFixedThreadPool(10);
-
- try {
- List<Future<?>> futs = new ArrayList<>();
-
- for (final IgniteCache cache : caches) {
- futs.add(executor.submit(new Runnable() {
- @Override public void run() {
- loadCache(cache.getName());
- }
- }));
- }
-
- for (Future<?> fut : futs)
- fut.get();
- }
- finally {
- executor.shutdown();
- }
- }
- else
- loadCache(caches.get(0).getName());
- }
+ loadCachesData();
}
- /**
- * @param cacheName Cache name.
- */
- private void loadCache(String cacheName) {
- println(cfg, "Loading data for cache: " + cacheName);
-
- long start = System.nanoTime();
-
+ /** {@inheritDoc} */
+ @Override protected void loadCacheData(String cacheName) {
try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cacheName)) {
for (int i = 0; i < args.preloadAmount(); i++) {
dataLdr.addData(i, new SampleValue(i));
@@ -92,9 +54,6 @@ public class IgniteGetBenchmark extends IgniteCacheAbstractBenchmark<Integer, Ob
}
}
}
-
- println(cfg, "Finished populating query data [cache=" + cacheName +
- ", time=" + ((System.nanoTime() - start) / 1_000_000) + "ms]");
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/9585dd32/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
index a50774f..d31a617 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryBenchmark.java
@@ -37,20 +37,22 @@ public class IgniteSqlQueryBenchmark extends IgniteCacheAbstractBenchmark<Intege
@Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
super.setUp(cfg);
- println(cfg, "Populating query data...");
+ loadCachesData();
+ }
- long start = System.nanoTime();
+ /** {@inheritDoc} */
+ @Override protected void loadCacheData(String cacheName) {
+ try (IgniteDataStreamer<Integer, Person> dataLdr = ignite().dataStreamer(cacheName)) {
+ for (int i = 0; i < args.range(); i++) {
+ if (i % 100 == 0 && Thread.currentThread().isInterrupted())
+ break;
- try (IgniteDataStreamer<Integer, Person> dataLdr = ignite().dataStreamer(cache.getName())) {
- for (int i = 0; i < args.range() && !Thread.currentThread().isInterrupted(); i++) {
dataLdr.addData(i, new Person(i, "firstName" + i, "lastName" + i, i * 1000));
if (i % 100000 == 0)
println(cfg, "Populated persons: " + i);
}
}
-
- println(cfg, "Finished populating query data in " + ((System.nanoTime() - start) / 1_000_000) + " ms.");
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/9585dd32/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryDistributedJoinBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryDistributedJoinBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryDistributedJoinBenchmark.java
index c5fdeb2..97aa38f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryDistributedJoinBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryDistributedJoinBenchmark.java
@@ -47,14 +47,23 @@ public class IgniteSqlQueryDistributedJoinBenchmark extends IgniteCacheAbstractB
println(cfg, "Populating query data...");
- long start = System.nanoTime();
-
range = args.range();
if (range <= 0)
throw new IllegalArgumentException();
- try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cache.getName())) {
+ println(cfg, "Populating join query data [orgCnt=" + range +
+ ", personCnt=" + range +
+ ", broadcastJoin=" + broadcast + "]");
+
+ loadCachesData();
+
+ executeQueryJoin(0, broadcast, true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void loadCacheData(String cacheName) {
+ try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cacheName)) {
for (int orgId = 0; orgId < range; orgId++) {
dataLdr.addData(orgId, new Organization(orgId, "org" + orgId));
@@ -73,13 +82,6 @@ public class IgniteSqlQueryDistributedJoinBenchmark extends IgniteCacheAbstractB
dataLdr.close();
}
-
- println(cfg, "Finished populating join query [orgCnt=" + range +
- ", personCnt=" + range +
- ", broadcastJoin=" + broadcast +
- ", time=" + ((System.nanoTime() - start) / 1_000_000) + "ms]");
-
- executeQueryJoin(0, broadcast, true);
}
/**
@@ -141,6 +143,8 @@ public class IgniteSqlQueryDistributedJoinBenchmark extends IgniteCacheAbstractB
qry.setDistributedJoins(true);
qry.setArgs(orgId);
+ IgniteCache<Integer, Object> cache = cacheForOperation();
+
if (planOnly) {
String plan = (String)cache.query(qry).getAll().get(0).get(0);
http://git-wip-us.apache.org/repos/asf/ignite/blob/9585dd32/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
index 1f8006d..79b336b 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryJoinBenchmark.java
@@ -40,9 +40,15 @@ public class IgniteSqlQueryJoinBenchmark extends IgniteCacheAbstractBenchmark<In
println(cfg, "Populating query data...");
- long start = System.nanoTime();
+ loadCachesData();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void loadCacheData(String cacheName) {
+ if (args.range() < 100)
+ throw new IllegalArgumentException("Invalid range: " + args.range());
- try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cache.getName())) {
+ try (IgniteDataStreamer<Object, Object> dataLdr = ignite().dataStreamer(cacheName)) {
final int orgRange = args.range() / 10;
// Populate organizations.
@@ -61,8 +67,6 @@ public class IgniteSqlQueryJoinBenchmark extends IgniteCacheAbstractBenchmark<In
println(cfg, "Populated persons: " + i);
}
}
-
- println(cfg, "Finished populating join query data in " + ((System.nanoTime() - start) / 1_000_000) + " ms.");
}
/** {@inheritDoc} */
@@ -100,6 +104,8 @@ public class IgniteSqlQueryJoinBenchmark extends IgniteCacheAbstractBenchmark<In
* @throws Exception If failed.
*/
private Collection<List<?>> executeQueryJoin(double minSalary, double maxSalary) throws Exception {
+ IgniteCache<Integer, Object> cache = cacheForOperation();
+
SqlFieldsQuery qry = new SqlFieldsQuery(
"select p.id, p.orgId, p.firstName, p.lastName, p.salary, o.name " +
"from Person p " +
http://git-wip-us.apache.org/repos/asf/ignite/blob/9585dd32/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
index dfa4cbc..9921ef7 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutBenchmark.java
@@ -25,7 +25,6 @@ import javax.cache.Cache;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.yardstick.cache.model.Person;
-import org.yardstickframework.BenchmarkConfiguration;
import static org.yardstickframework.BenchmarkUtils.println;
@@ -40,14 +39,11 @@ public class IgniteSqlQueryPutBenchmark extends IgniteCacheAbstractBenchmark<Int
private AtomicInteger qryCnt = new AtomicInteger();
/** {@inheritDoc} */
- @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
- super.setUp(cfg);
- }
-
- /** {@inheritDoc} */
@Override public boolean test(Map<Object, Object> ctx) throws Exception {
ThreadLocalRandom rnd = ThreadLocalRandom.current();
+ IgniteCache<Integer, Object> cache = cacheForOperation();
+
if (rnd.nextBoolean()) {
double salary = rnd.nextDouble() * args.range() * 1000;
http://git-wip-us.apache.org/repos/asf/ignite/blob/9585dd32/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutSeparatedBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutSeparatedBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutSeparatedBenchmark.java
index b74978e..d18be0d 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutSeparatedBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteSqlQueryPutSeparatedBenchmark.java
@@ -55,6 +55,8 @@ public class IgniteSqlQueryPutSeparatedBenchmark extends IgniteCacheAbstractBenc
}
}
else {
+ IgniteCache<Integer, Object> cache = cacheForOperation();
+
int i = rnd.nextInt(args.range());
cache.put(i, new Person(i, "firstName" + i, "lastName" + i, i * 1000));
@@ -70,6 +72,8 @@ public class IgniteSqlQueryPutSeparatedBenchmark extends IgniteCacheAbstractBenc
* @throws Exception If failed.
*/
private Collection<Cache.Entry<Integer, Object>> executeQuery(double minSalary, double maxSalary) throws Exception {
+ IgniteCache<Integer, Object> cache = cacheForOperation();
+
SqlQuery qry = new SqlQuery(Person.class, "salary >= ? and salary <= ?");
qry.setArgs(minSalary, maxSalary);