You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2016/12/15 09:21:19 UTC
[13/18] ignite git commit: IGNITE-3292 Fixed "Yardstick: add logging
of preloading progress". This closes #1317.
IGNITE-3292 Fixed "Yardstick: add logging of preloading progress". This closes #1317.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7094c0fd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7094c0fd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7094c0fd
Branch: refs/heads/ignite-4371
Commit: 7094c0fd8dcf80ee17ca84e630565d11407f9c2c
Parents: cb1fd82
Author: oleg-ostanin <oo...@gridgain.com>
Authored: Wed Dec 14 15:20:51 2016 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Wed Dec 14 15:21:56 2016 +0300
----------------------------------------------------------------------
.../yardstick/IgniteAbstractBenchmark.java | 30 ++++
.../yardstick/IgniteBenchmarkArguments.java | 11 ++
.../ignite/yardstick/IgniteBenchmarkUtils.java | 42 ++++-
.../apache/ignite/yardstick/PreloadLogger.java | 155 +++++++++++++++++++
.../IgniteCacheRandomOperationBenchmark.java | 25 +--
5 files changed, 253 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
index fa93f00..522499a 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteAbstractBenchmark.java
@@ -39,6 +39,9 @@ public abstract class IgniteAbstractBenchmark extends BenchmarkDriverAdapter {
/** Arguments. */
protected final IgniteBenchmarkArguments args = new IgniteBenchmarkArguments();
+ /** Logger */
+ private PreloadLogger lgr;
+
/** Node. */
private IgniteNode node;
@@ -60,6 +63,33 @@ public abstract class IgniteAbstractBenchmark extends BenchmarkDriverAdapter {
waitForNodes();
}
+ /**
+ * Prints non-system caches sizes during preload.
+ *
+ * @param logInterval time interval between printing preload log. Required to be positive.
+ */
+ protected void startPreloadLogging(long logInterval) {
+ try {
+ if (node != null && cfg != null && logInterval >= 0)
+ lgr = IgniteBenchmarkUtils.startPreloadLogger(node, cfg, logInterval);
+ else
+ BenchmarkUtils.println("Failed to start preload logger [node=" + node + ", cfg = " + cfg +
+ ", logInterval = " + logInterval + "]");
+ }
+ catch (Exception e) {
+ BenchmarkUtils.error("Failed to start preload logger [node=" + node + ", cfg = " + cfg +
+ ", logInterval = " + logInterval + "]", e);
+ }
+ }
+
+ /**
+ * Terminates printing preload log.
+ */
+ protected void stopPreloadLogging() {
+ if (lgr != null)
+ lgr.stopAndPrintStatistics();
+ }
+
/** {@inheritDoc} */
@Override public void tearDown() throws Exception {
if (node != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 1854938..2d2da5a 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -101,6 +101,10 @@ public class IgniteBenchmarkArguments {
public int preloadAmount = 500_000;
/** */
+ @Parameter(names = {"-plfreq", "--preloadLogFrequency"}, description = "Interval between printing logs")
+ public long preloadLogsInterval = 30_000;
+
+ /** */
@Parameter(names = {"-j", "--jobs"}, description = "Number of jobs for compute benchmarks")
private int jobs = 10;
@@ -290,6 +294,13 @@ public class IgniteBenchmarkArguments {
}
/**
+ * @return Preload log printing interval in seconds.
+ */
+ public long preloadLogsInterval() {
+ return preloadLogsInterval;
+ }
+
+ /**
* @return Configuration file.
*/
public String configuration() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
index 07549d5..c86dadb 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtils.java
@@ -20,6 +20,11 @@ package org.apache.ignite.yardstick;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
import javax.cache.CacheException;
import org.apache.ignite.IgniteTransactions;
import org.apache.ignite.Ignition;
@@ -31,14 +36,30 @@ import org.apache.ignite.transactions.TransactionIsolation;
import org.apache.ignite.transactions.TransactionOptimisticException;
import org.apache.ignite.transactions.TransactionRollbackException;
import org.apache.ignite.yardstick.cache.IgnitePutBenchmark;
+import org.yardstickframework.BenchmarkConfiguration;
import org.yardstickframework.BenchmarkDriver;
import org.yardstickframework.BenchmarkDriverStartUp;
+import org.yardstickframework.BenchmarkUtils;
/**
* Utils.
*/
public class IgniteBenchmarkUtils {
/**
+ * Scheduler executor.
+ */
+ private static final ScheduledExecutorService exec =
+ Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
+ @Override public Thread newThread(Runnable run) {
+ Thread thread = Executors.defaultThreadFactory().newThread(run);
+
+ thread.setDaemon(true);
+
+ return thread;
+ }
+ });
+
+ /**
* Utility class constructor.
*/
private IgniteBenchmarkUtils() {
@@ -53,7 +74,7 @@ public class IgniteBenchmarkUtils {
* @throws Exception If failed.
*/
public static <T> T doInTransaction(IgniteTransactions igniteTx, TransactionConcurrency txConcurrency,
- TransactionIsolation txIsolation, Callable<T> clo) throws Exception {
+ TransactionIsolation txIsolation, Callable<T> clo) throws Exception {
while (true) {
try (Transaction tx = igniteTx.txStart(txConcurrency, txIsolation)) {
T res = clo.call();
@@ -141,4 +162,23 @@ public class IgniteBenchmarkUtils {
args.add(arg);
args.add(val.toString());
}
+
+ /**
+ * Prints non-system cache sizes during preload.
+ *
+ * @param node Ignite node.
+ * @param cfg Benchmark configuration.
+ * @param logsInterval Time interval in milliseconds between printing logs.
+ */
+ public static PreloadLogger startPreloadLogger(IgniteNode node, BenchmarkConfiguration cfg, long logsInterval) {
+ PreloadLogger lgr = new PreloadLogger(node, cfg);
+
+ ScheduledFuture<?> fut = exec.scheduleWithFixedDelay(lgr, 0L, logsInterval, TimeUnit.MILLISECONDS);
+
+ lgr.setFuture(fut);
+
+ BenchmarkUtils.println(cfg, "Preload logger was started.");
+
+ return lgr;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java
new file mode 100644
index 0000000..c14a1df
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/PreloadLogger.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Prints non-system caches size.
+ */
+public class PreloadLogger implements Runnable {
+ /** Benchmark configuration. */
+ private BenchmarkConfiguration cfg;
+
+ /** List of caches whose size to be printed during preload. */
+ private Collection<IgniteCache<Object, Object>> caches;
+
+ /** Map for keeping previous values to make sure all the caches are working correctly. */
+ private Map<String, Long> cntrs;
+
+ /** String template used in String.format() to make output readable. */
+ private String strFmt;
+
+ /** Future instance to stop print log. */
+ private ScheduledFuture<?> fut;
+
+ /**
+ * @param node Ignite node.
+ * @param cfg BenchmarkConfiguration.
+ */
+ public PreloadLogger(IgniteNode node, BenchmarkConfiguration cfg) {
+ this.caches = new ArrayList<>();
+ this.cntrs = new HashMap<>();
+ this.cfg = cfg;
+
+ init(node);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ printCachesStatistics();
+ }
+
+ /**
+ * Prints non-system cache sizes.
+ */
+ public synchronized void printCachesStatistics() {
+ for (IgniteCache<Object, Object> cache : caches) {
+ try {
+ printCacheStatistics(cache);
+ }
+ catch (Exception e) {
+ BenchmarkUtils.println(cfg, "Failed to print cache size [cache=" + cache.getName()
+ + ", msg=" + e.getMessage() + "]");
+ }
+ }
+ }
+
+ /**
+ * Print cache size along with amount of recently loaded entries.
+ *
+ * @param cache Ignite cache.
+ */
+ private void printCacheStatistics(IgniteCache<Object, Object> cache) {
+ String cacheName = cache.getName();
+
+ long cacheSize = cache.sizeLong();
+
+ long recentlyLoaded = cacheSize - cntrs.get(cacheName);
+ String recLoaded = recentlyLoaded == 0 ? String.valueOf(recentlyLoaded) : "+" + recentlyLoaded;
+
+ BenchmarkUtils.println(cfg, String.format(strFmt, cacheName, cacheSize, recLoaded));
+
+ cntrs.put(cacheName, cacheSize);
+ }
+
+ /**
+ * Helper method for initializing the cache list and the counters map.
+ *
+ * @param node Ignite node.
+ */
+ private void init(IgniteNode node) {
+ int longestName = 0;
+
+ for (String cacheName : node.ignite().cacheNames()) {
+ IgniteCache<Object, Object> cache = node.ignite().cache(cacheName);
+
+ caches.add(cache);
+
+ // Set up an initial values to the map.
+ cntrs.put(cache.getName(), 0L);
+
+ // Find out the length of the longest cache name.
+ longestName = Math.max(cache.getName().length(), longestName);
+ }
+
+ // Should look like "Preload:%-20s%-8d\t(%s)"
+ strFmt = "Preload:%-" + (longestName + 4) + "s%-8d\t(%s)";
+ }
+
+ /**
+ * Set future.
+ */
+ public void setFuture(ScheduledFuture<?> fut) {
+ this.fut = fut;
+ }
+
+ /**
+ * Terminates printing log.
+ */
+ public void stopAndPrintStatistics() {
+ try {
+ if (fut != null) {
+ if (!fut.cancel(true)) {
+ U.sleep(200);
+
+ if (!fut.cancel(true)) {
+ BenchmarkUtils.println(cfg, "Failed to cancel Preload logger.");
+
+ return;
+ }
+ }
+ }
+
+ printCachesStatistics();
+ }
+ catch (Exception e) {
+ BenchmarkUtils.error("Failed to stop Preload logger.", e);
+ }
+
+ BenchmarkUtils.println(cfg, "Preload logger was stopped.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7094c0fd/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
index d37cdca..590b64f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/load/IgniteCacheRandomOperationBenchmark.java
@@ -240,7 +240,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
configureCacheSqlDescriptor(cacheName, queryEntity, valCls);
}
- } catch (ClassNotFoundException e) {
+ }
+ catch (ClassNotFoundException e) {
BenchmarkUtils.println(e.getMessage());
BenchmarkUtils.println("This can be a BinaryObject. Ignoring exception.");
@@ -274,7 +275,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
throw new IgniteException("Class is unknown for the load test. Make sure you " +
"specified its full name [clsName=" + cacheTypeMetadata.getKeyType() + ']');
}
- } catch (ClassNotFoundException e) {
+ }
+ catch (ClassNotFoundException e) {
BenchmarkUtils.println(e.getMessage());
BenchmarkUtils.println("This can be a BinaryObject. Ignoring exception.");
@@ -401,8 +403,10 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
*/
private void preLoading() throws Exception {
if (args.preloadAmount() > args.range())
- throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") must by less then the" +
- " range (\"-r\", \"--range\").");
+ throw new IllegalArgumentException("Preloading amount (\"-pa\", \"--preloadAmount\") " +
+ "must by less then the range (\"-r\", \"--range\").");
+
+ startPreloadLogging(args.preloadLogsInterval());
Thread[] threads = new Thread[availableCaches.size()];
@@ -423,6 +427,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
for (Thread thread : threads)
thread.join();
+
+ stopPreloadLogging();
}
/**
@@ -641,8 +647,8 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
* @param map Parameters map.
*/
private void updateStat(Map<Object, Object> map) {
- for (Operation op: Operation.values())
- for (String cacheName: ignite().cacheNames()) {
+ for (Operation op : Operation.values())
+ for (String cacheName : ignite().cacheNames()) {
String opCacheKey = op + "_" + cacheName;
Integer val = (Integer)map.get(opCacheKey);
@@ -658,6 +664,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
/**
* Execute operations in transaction.
+ *
* @param map Parameters map.
* @throws Exception if fail.
*/
@@ -929,7 +936,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
Integer[] sub = new Integer[cnt];
- for (int i = 0; i< cnt; i++)
+ for (int i = 0; i < cnt; i++)
sub[i] = nextRandom(args.range());
sql = String.format(sql, sub);
@@ -994,7 +1001,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
/** {@inheritDoc} */
@Override public boolean evaluate(CacheEntryEvent evt) throws CacheEntryListenerException {
- return flag =! flag;
+ return flag = !flag;
}
}
@@ -1230,7 +1237,7 @@ public class IgniteCacheRandomOperationBenchmark extends IgniteAbstractBenchmark
private final boolean distributedJoin;
/**
- * @param sql SQL.
+ * @param sql SQL.
* @param distributedJoin Distributed join flag.
*/
public TestQuery(String sql, boolean distributedJoin) {