You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2016/10/10 12:13:38 UTC

[42/50] [abbrv] ignite git commit: io test

io test


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8d0f6c73
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8d0f6c73
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8d0f6c73

Branch: refs/heads/ignite-gg-8-io2-selNow
Commit: 8d0f6c73d156cd8f7722bce4ffb6f0797925e500
Parents: 3d2e47c
Author: sboikov <sb...@gridgain.com>
Authored: Tue Oct 4 15:06:02 2016 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Oct 4 15:06:02 2016 +0300

----------------------------------------------------------------------
 .../config/ignite-localhost-config-srv.xml      |  62 +++++
 .../config/ignite-localhost-config2.xml         |  61 +++++
 .../yardstick/IgniteBenchmarkUtilsTmp.java      | 103 ++++++++
 .../MyBenchmarkProbePointCsvWriter.java         | 250 +++++++++++++++++++
 .../yardstick/MyThroughputLatencyProbe.java     | 195 +++++++++++++++
 5 files changed, 671 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/8d0f6c73/modules/yardstick/config/ignite-localhost-config-srv.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-localhost-config-srv.xml b/modules/yardstick/config/ignite-localhost-config-srv.xml
new file mode 100644
index 0000000..61ef5b8
--- /dev/null
+++ b/modules/yardstick/config/ignite-localhost-config-srv.xml
@@ -0,0 +1,62 @@
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+    <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+        <property name="localHost" value="127.0.0.1"/>
+
+        <!--
+        <property name="systemThreadPoolSize" value="1"/>
+        <property name="rebalanceThreadPoolSize" value="1"/>
+        -->
+
+        <property name="cacheConfiguration">
+            <list>
+            </list>
+        </property>
+
+        <property name="communicationSpi">
+            <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+                <property name="selectorsCount" value="2"/>
+                <property name="connectionsPerNode" value="1"/>
+            </bean>
+        </property>
+
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <value>127.0.0.1:47500</value>
+                                <value>127.0.0.1:47501</value>
+                                <value>127.0.0.1:47502</value>
+                                <value>127.0.0.1:47503</value>
+                                <value>127.0.0.1:47504</value>
+                                <value>127.0.0.1:47505</value>
+                                <value>127.0.0.1:47506</value>
+                                <value>127.0.0.1:47507</value>
+                                <value>127.0.0.1:47508</value>
+                                <value>127.0.0.1:47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+
+        <property name="peerClassLoadingEnabled" value="false"/>
+
+        <property name="metricsLogFrequency" value="5000"/>
+
+        <property name="includeEventTypes">
+            <list/>
+        </property>
+
+        <property name="loadBalancingSpi">
+            <bean class="org.apache.ignite.spi.loadbalancing.roundrobin.RoundRobinLoadBalancingSpi">
+                <property name="perTask" value="false"/>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/8d0f6c73/modules/yardstick/config/ignite-localhost-config2.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-localhost-config2.xml b/modules/yardstick/config/ignite-localhost-config2.xml
new file mode 100644
index 0000000..620b78d
--- /dev/null
+++ b/modules/yardstick/config/ignite-localhost-config2.xml
@@ -0,0 +1,61 @@
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="
+        http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+    <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
+        <property name="localHost" value="127.0.0.1"/>
+
+        <!--
+        <property name="systemThreadPoolSize" value="1"/>
+        <property name="rebalanceThreadPoolSize" value="1"/>
+        -->
+
+        <property name="cacheConfiguration">
+            <list>
+            </list>
+        </property>
+
+        <property name="communicationSpi">
+            <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+                <property name="selectorsCount" value="16"/>
+            </bean>
+        </property>
+
+        <property name="discoverySpi">
+            <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+                <property name="ipFinder">
+                    <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+                        <property name="addresses">
+                            <list>
+                                <value>127.0.0.1:47500</value>
+                                <value>127.0.0.1:47501</value>
+                                <value>127.0.0.1:47502</value>
+                                <value>127.0.0.1:47503</value>
+                                <value>127.0.0.1:47504</value>
+                                <value>127.0.0.1:47505</value>
+                                <value>127.0.0.1:47506</value>
+                                <value>127.0.0.1:47507</value>
+                                <value>127.0.0.1:47508</value>
+                                <value>127.0.0.1:47509</value>
+                            </list>
+                        </property>
+                    </bean>
+                </property>
+            </bean>
+        </property>
+
+        <property name="peerClassLoadingEnabled" value="false"/>
+
+        <property name="metricsLogFrequency" value="5000"/>
+
+        <property name="includeEventTypes">
+            <list/>
+        </property>
+
+        <property name="loadBalancingSpi">
+            <bean class="org.apache.ignite.spi.loadbalancing.roundrobin.RoundRobinLoadBalancingSpi">
+                <property name="perTask" value="false"/>
+            </bean>
+        </property>
+    </bean>
+</beans>

http://git-wip-us.apache.org/repos/asf/ignite/blob/8d0f6c73/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtilsTmp.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtilsTmp.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtilsTmp.java
new file mode 100644
index 0000000..7ca6c74
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkUtilsTmp.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.yardstick.cache.IgniteIoTestSendAllBenchmark;
+import org.apache.ignite.yardstick.cache.IgniteIoTestSendRandomBenchmark;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkDriverStartUp;
+
+/**
+ * Utils.
+ */
+public class IgniteBenchmarkUtilsTmp {
+    /**
+     * Starts nodes/driver in single JVM for quick benchmarks testing.
+     *
+     * -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=delay=60s,duration=60s,filename=myrecording.jfr -XX:FlightRecorderOptions=defaultrecording=true
+     * -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=delay=60s,duration=60s,filename=myrecording.jfr -XX:FlightRecorderOptions=defaultrecording=true
+     * @param args Command line arguments.
+     * @throws Exception If failed.
+     */
+    public static void main(String[] args) throws Exception {
+        final Class<? extends BenchmarkDriver> benchmark = IgniteIoTestSendRandomBenchmark.class;
+//        final Class<? extends BenchmarkDriver> benchmark = IgniteIoTestSendAllBenchmark.class;
+
+        final int threads = 16;
+
+        final boolean clientDriverNode = true;
+
+        final int extraNodes = 4;
+
+        final int warmUp = 30;
+        final int duration = 60;
+
+        final int range = 1_000_000;
+
+        final boolean throughputLatencyProbe = true;
+
+        for (int i = 0; i < extraNodes; i++) {
+            final String cfg = "modules/yardstick/config/ignite-localhost-config-srv.xml";
+
+            IgniteConfiguration nodeCfg = Ignition.loadSpringBean(cfg, "grid.cfg");
+
+            nodeCfg.setGridName("node-" + i);
+            nodeCfg.setMetricsLogFrequency(0);
+
+            Ignition.start(nodeCfg);
+        }
+
+        final String cfg = "modules/yardstick/config/ignite-localhost-config2.xml";
+
+        ArrayList<String> args0 = new ArrayList<>();
+
+        addArg(args0, "-t", threads);
+        addArg(args0, "-w", warmUp);
+        addArg(args0, "-d", duration);
+        addArg(args0, "-r", range);
+        addArg(args0, "-dn", benchmark.getSimpleName());
+        addArg(args0, "-sn", "IgniteNode");
+        addArg(args0, "-cfg", cfg);
+
+        if (throughputLatencyProbe) {
+            addArg(args0, "-pr", "ThroughputLatencyProbe");
+            addArg(args0, "--writer", "MyBenchmarkProbePointCsvWriter");
+        }
+
+        if (clientDriverNode)
+            args0.add("-cl");
+
+        BenchmarkDriverStartUp.main(args0.toArray(new String[args0.size()]));
+    }
+
+    /**
+     * @param args Arguments.
+     * @param arg Argument name.
+     * @param val Argument value.
+     */
+    private static void addArg(List<String> args, String arg, Object val) {
+        args.add(arg);
+        args.add(val.toString());
+    }
+}
+
+// -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=delay=40s,duration=30s,filename=myrecording.jfr -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true

http://git-wip-us.apache.org/repos/asf/ignite/blob/8d0f6c73/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyBenchmarkProbePointCsvWriter.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyBenchmarkProbePointCsvWriter.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyBenchmarkProbePointCsvWriter.java
new file mode 100644
index 0000000..d2f474d
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyBenchmarkProbePointCsvWriter.java
@@ -0,0 +1,250 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+package org.apache.ignite.yardstick;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Locale;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkProbe;
+import org.yardstickframework.BenchmarkProbePoint;
+import org.yardstickframework.BenchmarkProbePointWriter;
+import org.yardstickframework.BenchmarkUtils;
+
+import static org.yardstickframework.BenchmarkUtils.WEIGHT_DELIMITER;
+import static org.yardstickframework.BenchmarkUtils.fixFolderName;
+
+/**
+ * CSV probe point writer.
+ */
+public class MyBenchmarkProbePointCsvWriter implements BenchmarkProbePointWriter {
+    /** */
+    private static final String DUPLICATE_TO_OUTPUT = "BENCHMARK_WRITER_DUPLICATE_TO_OUTPUT";
+
+    /** */
+    private static final boolean DEFAULT_DUPLICATE_TO_OUTPUT = false;
+
+    /** */
+    public static final String META_INFO_SEPARATOR = ",";
+
+    /** */
+    public static final String DRV_NAMES_PREFIX = "@@";
+
+    /** */
+    public static final String META_INFO_PREFIX = "**";
+
+    /** */
+    public static final SimpleDateFormat FORMAT = new SimpleDateFormat("yyyyMMdd-HHmmss");
+
+    /** */
+    private PrintWriter writer;
+
+    /** */
+    private BenchmarkConfiguration cfg;
+
+    /** */
+    private BenchmarkDriver drv;
+
+    /** */
+    private long startTime;
+
+    /** */
+    private boolean dupToOutput;
+
+    /** */
+    private File outPath;
+
+    private double sum;
+
+    private long cntr;
+
+    /** {@inheritDoc} */
+    @Override public void start(BenchmarkDriver drv, BenchmarkConfiguration cfg, long startTime) {
+        this.cfg = cfg;
+        this.drv = drv;
+        this.startTime = startTime;
+
+        dupToOutput = duplicateToOutput(cfg);
+
+        String path = cfg.outputFolder();
+
+        File folder = null;
+
+        if (path != null) {
+            folder = new File(path);
+
+            if (!folder.exists()) {
+                if (!folder.mkdirs())
+                    throw new IllegalStateException("Can not create folder: " + folder.getAbsolutePath());
+            }
+        }
+
+        String desc = drv.description() == null ? "" : drv.description();
+
+        desc = desc.replaceAll("-+", "-").replaceAll(",|\\\\|/|\\||%|:|<|>|\\*|\\?|\"|\\s", "-");
+
+        desc = desc.charAt(0) == '-' ? desc : '-' + desc;
+
+        String subFolderName = FORMAT.format(new Date(startTime));
+
+        String hostName = cfg.hostName().isEmpty() ? "" : '-' + cfg.hostName();
+
+        if (cfg.driverNames().size() > 1) {
+            StringBuilder sb = new StringBuilder();
+
+            for (String drvName : cfg.driverNames())
+                sb.append(drvName.split(WEIGHT_DELIMITER)[0].trim()).append('-');
+
+            if (sb.length() > 0)
+                sb.delete(sb.length() - 1, sb.length());
+
+            subFolderName += '-' + sb.toString();
+
+            subFolderName += File.separator + desc.substring(1) + hostName;
+        }
+        else
+            subFolderName += desc + hostName;
+
+        subFolderName = fixFolderName(subFolderName);
+
+        outPath = folder == null ? new File(subFolderName) : new File(folder, subFolderName);
+
+        if (!outPath.exists()) {
+            if (!outPath.mkdirs())
+                throw new IllegalStateException("Can not create folder: " + outPath.getAbsolutePath());
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writePoints(BenchmarkProbe probe, Collection<BenchmarkProbePoint> points) throws Exception {
+        if (writer == null) {
+            String fileName = probe.getClass().getSimpleName() + ".csv";
+
+            File f = outPath == null ? new File(fileName) : new File(outPath, fileName);
+
+            writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(f)));
+
+            String parent;
+
+            // Multiple drivers.
+            if (cfg.driverNames().size() > 1) {
+                File outPath0 = outPath.getParentFile();
+
+                parent = outPath0.getParent() == null ? outPath.getPath() : outPath0.getPath();
+            }
+            else
+                parent = outPath.getParent() == null ? outPath.getPath() : outPath.getParent();
+
+            BenchmarkUtils.println(cfg, probe.getClass().getSimpleName() + " results will be saved to: " + parent);
+
+            println("--Probe dump file for probe: " + probe + " (" + probe.getClass() + ")");
+            println("--Created " + new Date(startTime));
+            println("--Benchmark config: " + removeUnwantedChars(cfg.toString()));
+            println("--Description: " + removeUnwantedChars(drv.description() == null ? "" : drv.description()));
+            println(DRV_NAMES_PREFIX + cfg.driverNames().toString().replaceAll("\\[", "").replaceAll("]", ""));
+
+            if (probe.metaInfo() != null && !probe.metaInfo().isEmpty()) {
+                print(META_INFO_PREFIX);
+
+                int i = 0;
+
+                for (String metaInfo : probe.metaInfo())
+                    print("\"" + metaInfo + "\"" + (++i == probe.metaInfo().size() ? "" : META_INFO_SEPARATOR));
+
+                if (i != 0)
+                    println("");
+            }
+        }
+
+        for (BenchmarkProbePoint pt : points) {
+            print(String.valueOf(pt.time()));
+            print(",");
+
+            double[] vals = pt.values();
+
+            for (int i = 0; i < vals.length; i++) {
+                if (i == 0) {
+                    sum += vals[i];
+                    cntr++;
+                }
+
+                print(String.format(Locale.US, "%.2f", vals[i]));
+
+                if (i != vals.length - 1)
+                    print(",");
+            }
+
+            println("");
+        }
+
+        writer.flush();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws Exception {
+        if (writer != null) {
+            print("Average: " + sum / cntr);
+
+            writer.close();
+        }
+    }
+
+    /**
+     * @param s String to write.
+     */
+    private void print(String s) {
+        writer.print(s);
+
+        if (dupToOutput)
+            cfg.output().print(s);
+    }
+
+    /**
+     * @param s String to write.
+     */
+    private void println(String s) {
+        writer.println(s);
+
+        if (dupToOutput)
+            BenchmarkUtils.println(cfg, s);
+    }
+
+    /**
+     * @param val String.
+     * @return String with removed chars.
+     */
+    private String removeUnwantedChars(String val) {
+        return val.replaceAll("\n|\t|\r|\f", "");
+    }
+
+    /**
+     * @param cfg Config.
+     * @return Flat indicating whether to duplicate to output or not.
+     */
+    private boolean duplicateToOutput(BenchmarkConfiguration cfg) {
+        try {
+            return Boolean.parseBoolean(cfg.customProperties().get(DUPLICATE_TO_OUTPUT));
+        }
+        catch (NumberFormatException | NullPointerException ignored) {
+            return DEFAULT_DUPLICATE_TO_OUTPUT;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/8d0f6c73/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyThroughputLatencyProbe.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyThroughputLatencyProbe.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyThroughputLatencyProbe.java
new file mode 100644
index 0000000..3df8dc2
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/MyThroughputLatencyProbe.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkExecutionAwareProbe;
+import org.yardstickframework.BenchmarkProbePoint;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Probe that calculates throughput and average latency.
+ */
+public class MyThroughputLatencyProbe implements BenchmarkExecutionAwareProbe {
+    /** Operations executed. */
+    private MyThroughputLatencyProbe.ThreadAgent[] agents;
+
+    /** Collected points. */
+    private Collection<BenchmarkProbePoint> collected = new ArrayList<>();
+
+    /** Service building probe points. */
+    private ExecutorService buildingService;
+
+    /** */
+    private BenchmarkConfiguration cfg;
+
+    /** Last data collection time stamp. */
+    private volatile long lastTstamp;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("BusyWait")
+    @Override public void start(BenchmarkDriver drv, BenchmarkConfiguration cfg) throws Exception {
+        this.cfg = cfg;
+
+        agents = new MyThroughputLatencyProbe.ThreadAgent[cfg.threads()];
+
+        for (int i = 0; i < agents.length; i++)
+            agents[i] = new MyThroughputLatencyProbe.ThreadAgent();
+
+        buildingService = Executors.newSingleThreadExecutor();
+
+        println(cfg, getClass().getSimpleName() + " is started.");
+
+        lastTstamp = System.currentTimeMillis();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void stop() throws Exception {
+        if (buildingService != null) {
+            buildingService.shutdownNow();
+
+            buildingService.awaitTermination(1, MINUTES);
+
+            println(cfg, getClass().getSimpleName() + " is stopped.");
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<String> metaInfo() {
+        return Arrays.asList("Time, sec", "Operations/sec (more is better)", "Latency, nsec (less is better)");
+    }
+
+    /** {@inheritDoc} */
+    @Override public synchronized Collection<BenchmarkProbePoint> points() {
+        Collection<BenchmarkProbePoint> ret = collected;
+
+        collected = new ArrayList<>(ret.size() + 5);
+
+        double sum = 0;
+
+        for (BenchmarkProbePoint p : ret)
+            sum += p.values()[0];
+
+        System.out.println("Average: " + sum / ret.size());
+
+        return ret;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void buildPoint(final long time) {
+        buildingService.execute(new Runnable() {
+            @Override public void run() {
+                long lastTstamp0 = lastTstamp;
+
+                long lastTstamp1 = System.currentTimeMillis();
+
+                lastTstamp = lastTstamp1;
+
+                // Time delta in seconds, rounding is used because Thread.sleep(1000) can last less than a second.
+                long delta = (long)Math.floor((lastTstamp1 - lastTstamp0) / 1000d + 0.5);
+
+                MyThroughputLatencyProbe.ThreadAgent collector = new MyThroughputLatencyProbe.ThreadAgent();
+
+                for (MyThroughputLatencyProbe.ThreadAgent agent : agents)
+                    agent.collect(collector);
+
+                double latency = collector.execCnt == 0 ? 0 : (double)collector.totalLatency / collector.execCnt;
+
+                if (delta != 0) {
+                    BenchmarkProbePoint pnt = new BenchmarkProbePoint(
+                        TimeUnit.MILLISECONDS.toSeconds(time),
+                        new double[] { delta == 0 ? Double.NaN : (double)collector.execCnt / delta, latency});
+
+                    collectPoint(pnt);
+                }
+            }
+        });
+    }
+
+    /**
+     * @param pnt Probe point.
+     */
+    private synchronized void collectPoint(BenchmarkProbePoint pnt) {
+        collected.add(pnt);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void beforeExecute(int threadIdx) {
+        agents[threadIdx].beforeExecute();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void afterExecute(int threadIdx) {
+        agents[threadIdx].afterExecute();
+    }
+
+    /**
+     *
+     */
+    private static class ThreadAgent {
+        /** Total execution count by thread. */
+        private long execCnt;
+
+        /** Total latency by  */
+        private long totalLatency;
+
+        /** Last before execute timestamp. */
+        private long beforeTs;
+
+        /**
+         *
+         */
+        public void beforeExecute() {
+            beforeTs = System.nanoTime();
+        }
+
+        /**
+         *
+         */
+        public void afterExecute() {
+            long latency = System.nanoTime() - beforeTs;
+
+            beforeTs = 0;
+
+            synchronized (this) {
+                execCnt++;
+                totalLatency += latency;
+            }
+        }
+
+        /**
+         * @param other Thread agent.
+         */
+        public synchronized void collect(MyThroughputLatencyProbe.ThreadAgent other) {
+            other.execCnt += execCnt;
+            other.totalLatency += totalLatency;
+
+            execCnt = 0;
+            totalLatency = 0;
+        }
+    }
+}