You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/03/19 06:26:55 UTC
[18/50] [abbrv] ignite git commit: IGNITE-7531: Data load benchmarks.
This closes #3571.
IGNITE-7531: Data load benchmarks. This closes #3571.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ac3d5450
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ac3d5450
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ac3d5450
Branch: refs/heads/ignite-zk
Commit: ac3d545045037446014b66108364bbef39362847
Parents: 1142a36
Author: Pavel Kuznetsov <pa...@gmail.com>
Authored: Tue Mar 13 15:46:36 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Tue Mar 13 15:46:36 2018 +0300
----------------------------------------------------------------------
.../ignite-localhost-persistence-config.xml | 71 +++++++
.../benchmark-jdbc-thin-inmemory.properties | 108 +++++++++++
.../benchmark-jdbc-thin-persistence.properties | 113 +++++++++++
.../yardstick/IgniteBenchmarkArguments.java | 9 +-
.../yardstick/jdbc/AbstractJdbcBenchmark.java | 23 ++-
.../upload/AbstractNativeBenchmark.java | 114 +++++++++++
.../upload/AbstractUploadBenchmark.java | 187 +++++++++++++++++++
.../upload/BatchedInsertBenchmark.java | 65 +++++++
.../ignite/yardstick/upload/CopyBenchmark.java | 125 +++++++++++++
.../yardstick/upload/InsertBenchmark.java | 52 ++++++
.../yardstick/upload/NativePutBenchmark.java | 39 ++++
.../upload/NativeStreamerBenchmark.java | 66 +++++++
.../upload/UploadBenchmarkArguments.java | 158 ++++++++++++++++
.../yardstick/upload/model/QueryFactory.java | 181 ++++++++++++++++++
.../ignite/yardstick/upload/model/Values10.java | 76 ++++++++
.../org/yardstickframework/package-info.java | 25 +++
.../probes/TotalTimeProbe.java | 137 ++++++++++++++
17 files changed, 1543 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/ignite-localhost-persistence-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-localhost-persistence-config.xml b/modules/yardstick/config/ignite-localhost-persistence-config.xml
new file mode 100644
index 0000000..2e9d9ff
--- /dev/null
+++ b/modules/yardstick/config/ignite-localhost-persistence-config.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!--
+ Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+ <import resource="ignite-base-config.xml"/>
+
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" parent="base-ignite.cfg">
+ <property name="localHost" value="127.0.0.1"/>
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <value>127.0.0.1:47500</value>
+ <value>127.0.0.1:47501</value>
+ <value>127.0.0.1:47502</value>
+ <value>127.0.0.1:47503</value>
+ <value>127.0.0.1:47504</value>
+ <value>127.0.0.1:47505</value>
+ <value>127.0.0.1:47506</value>
+ <value>127.0.0.1:47507</value>
+ <value>127.0.0.1:47508</value>
+ <value>127.0.0.1:47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ <property name="dataStorageConfiguration" >
+ <bean class="org.apache.ignite.configuration.DataStorageConfiguration">
+ <property name="defaultDataRegionConfiguration">
+ <bean class="org.apache.ignite.configuration.DataRegionConfiguration">
+ <property name="persistenceEnabled" value="true"/>
+ </bean>
+ </property>
+ </bean>
+ </property>
+
+ <property name="communicationSpi">
+ <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi">
+ <property name="sharedMemoryPort" value="-1"/>
+ </bean>
+ </property>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
new file mode 100644
index 0000000..cacf726
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-inmemory.properties
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload in inmemory mode (persistence disabled).
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-inmemory-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --upload-jdbc-batch-size 1000 \
+"
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
new file mode 100644
index 0000000..b72e9b7
--- /dev/null
+++ b/modules/yardstick/config/upload/benchmark-jdbc-thin-persistence.properties
@@ -0,0 +1,113 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for data upload with enabled persistence.
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=TotalTimeProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=0
+
+# Threads count.
+t=1
+
+# Sync mode.
+sm=FULL_SYNC
+
+# Whither or not turn off WAL just for upload period.
+switchWal=false
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run only one time, warmup parameter is set to 0 due to custom warmup operation.
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativePutBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn NativeStreamerBenchmark -sn IgniteNode -ds ${ver}sql-upload-native-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --streamer-local-batch-size 1000, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn CopyBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn InsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --disable-wal ${switchWal}, \
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-persistence-config.xml -nn ${nodesNum} -b ${b} --warmup ${w} --operations 1 \
+ -jdbc jdbc:ignite:thin://auto.find/ \
+ --threads ${t} \
+ --syncMode ${sm} -dn BatchedInsertBenchmark -sn IgniteNode -ds ${ver}sql-upload-persistence-jdbc-thin-r1-${b}-backup \
+ --upload-rows 1000000 -cl --clientNodesAfterId 0 \
+ --upload-jdbc-batch-size 1000 \
+ --disable-wal ${switchWal} \
+"
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 8c453ad..9789aea 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -18,6 +18,8 @@
package org.apache.ignite.yardstick;
import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import java.util.Collections;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -29,6 +31,7 @@ import org.apache.ignite.transactions.TransactionIsolation;
import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.yardstick.cache.IgniteStreamerBenchmark;
+import org.apache.ignite.yardstick.upload.UploadBenchmarkArguments;
import org.jetbrains.annotations.Nullable;
/**
@@ -262,6 +265,10 @@ public class IgniteBenchmarkArguments {
@GridToStringInclude
private int clientNodesAfterId = -1;
+ @ParametersDelegate
+ @GridToStringInclude
+ public UploadBenchmarkArguments upload = new UploadBenchmarkArguments();
+
/**
* @return {@code True} if need set {@link DataStorageConfiguration}.
*/
@@ -384,7 +391,7 @@ public class IgniteBenchmarkArguments {
/**
* @return {@code True} if flag for native benchmarking is set.
*/
- public boolean isNative(){
+ public boolean isNative() {
return ntv;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
index 9ef8a77..6ab024f 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/AbstractJdbcBenchmark.java
@@ -37,18 +37,18 @@ import static org.yardstickframework.BenchmarkUtils.println;
/**
* JDBC benchmark that performs query operations.
*/
-abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
+public abstract class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
/** All {@link Connection}s associated with threads. */
private final List<Connection> threadConnections = new ArrayList<>();
/** JDBC URL. */
- private String url;
+ protected String url;
/** Each connection is also a transaction, so we better pin them to threads. */
protected ThreadLocal<Connection> conn = new ThreadLocal<Connection>() {
@Override protected Connection initialValue() {
try {
- Connection conn = connection();
+ Connection conn = connection(url);
synchronized (threadConnections) {
threadConnections.add(conn);
@@ -66,6 +66,9 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
@Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
super.setUp(cfg);
+ // activate cluster if it is not auto activated
+ ignite().cluster().active(true);
+
if (url == null) {
if (args.jdbcUrl().startsWith(JdbcThinUtils.URL_PREFIX)) {
String addr = findThinAddress();
@@ -77,12 +80,22 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
println("Using jdbc url:" + url);
- fillData(cfg, (IgniteEx)ignite(), args.range());
+ setupData();
ignite().close();
}
/**
+ * Sets up test data
+ *
+ * Gets executed before local Ignite node is closed
+ * @throws Exception On error.
+ */
+ protected void setupData() throws Exception {
+ fillData(cfg, (IgniteEx)ignite(), args.range());
+ }
+
+ /**
* Find address of client node, that thin driver should use.
*
* @return Address for thin driver.
@@ -128,7 +141,7 @@ abstract public class AbstractJdbcBenchmark extends IgniteAbstractBenchmark {
* @return JDBC connection.
* @throws SQLException On error.
*/
- private Connection connection() throws SQLException {
+ protected final Connection connection(String url) throws SQLException {
println("JDBC connect to: " + url);
Connection conn = DriverManager.getConnection(url);
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
new file mode 100644
index 0000000..de1ad28
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractNativeBenchmark.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+import org.apache.ignite.yardstick.upload.model.Values10;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for benchmarks that perform upload using java api.
+ */
+public abstract class AbstractNativeBenchmark extends IgniteAbstractBenchmark {
+ /** Number of entries to be uploaded during warmup. */
+ private long insertRowsCnt;
+
+ /** Name of the {@link #cache} */
+ private String cacheName;
+
+ /** Cache method {@link test(Map)} uploads data to */
+ private IgniteCache<Long, Values10> cache;
+
+ /**
+ * Sets up benchmark: performs warmup on one cache and creates another for {@link #test(Map)} method.
+ *
+ * @param cfg Benchmark configuration.
+ * @throws Exception - on error.
+ */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ cacheName = this.getClass().getSimpleName();
+
+ insertRowsCnt = args.upload.uploadRowsCnt();
+
+ // Number of entries to be uploaded during test().
+ long warmupRowsCnt = args.upload.warmupRowsCnt();
+
+ // warmup
+ BenchmarkUtils.println(cfg, "Starting custom warmup.");
+ String warmupCacheName = cacheName + "Warmup";
+
+ try (IgniteCache<Long, Values10> warmupCache = ignite().createCache(warmupCacheName)) {
+ upload(warmupCacheName, warmupRowsCnt);
+ }
+ finally {
+ ignite().destroyCache(warmupCacheName);
+ }
+
+ BenchmarkUtils.println(cfg, "Custom warmup finished.");
+
+ // cache for benchmarked action
+ cache = ignite().createCache(cacheName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void tearDown() throws Exception {
+ try {
+ if (cache == null)
+ throw new IllegalStateException("Cache is null, probably an error during setUp or warmup");
+
+ long size = cache.sizeLong();
+
+ if (size != insertRowsCnt) {
+ String msg = "Incorrect cache size: [actual=" + size + ", expected=" + insertRowsCnt +"].";
+
+ BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+ throw new RuntimeException(msg);
+ }
+
+ cache.close();
+
+ ignite().destroyCache(cacheName);
+
+ }
+ catch (IgniteException ex) {
+ BenchmarkUtils.println(cfg, "Could not close or destroy cache: " + ex);
+
+ throw ex;
+ }
+ finally {
+ super.tearDown();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ upload(cacheName, insertRowsCnt);
+
+ return true;
+ }
+
+ /** Uploads {@param insertsCnt} to cache with name {@param cacheName} using java api. */
+ protected abstract void upload(String cacheName, long insertsCnt);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
new file mode 100644
index 0000000..81a93e2
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/AbstractUploadBenchmark.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.yardstick.jdbc.AbstractJdbcBenchmark;
+import org.apache.ignite.yardstick.upload.model.QueryFactory;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Base class for upload benchmarks.
+ * Designed to run test method one single time.
+ * Introduces custom warmup operation.
+ */
+public abstract class AbstractUploadBenchmark extends AbstractJdbcBenchmark {
+ /** Total inserts size. */
+ long insertRowsCnt;
+
+ /** Rows count to be inserted and deleted during warmup */
+ long warmupRowsCnt;
+
+ /** Factory that hides all the test data details. */
+ protected QueryFactory queries = new QueryFactory();
+
+ /** {@inheritDoc} */
+ @Override public final void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ insertRowsCnt = args.upload.uploadRowsCnt();
+ warmupRowsCnt = args.upload.warmupRowsCnt();
+
+ init();
+
+ // Perform warmup keeping in mind wal optimization.
+ BenchmarkUtils.println(this.cfg, "Starting custom warmup");
+
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+ try (Connection warmupConn = uploadConnection()) {
+ warmup(warmupConn);
+ }
+
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_ON_WAL);
+
+ BenchmarkUtils.println(this.cfg, "Custom warmup finished");
+
+ dropAndCreate();
+ }
+
+ /**
+ * Inits benchmark fields.
+ */
+ protected void init() {
+ // No-op.
+ }
+
+ /**
+ * Method to warm up Benchmark server. <br/>
+ * In upload benchmarks we need warmup action
+ * and real test action to be separated.
+ */
+ protected abstract void warmup(Connection warmupConn) throws Exception;
+
+ /**
+ * Creates empty table.
+ */
+ @Override protected void setupData() throws Exception{
+ dropAndCreate();
+ }
+
+ /**
+ * Uploads data using this special connection, that may have additional
+ * url parameters, such as {@code streaming=true}.
+ */
+ protected abstract void upload(Connection uploadConn) throws Exception;
+
+ /** {@inheritDoc} */
+ @Override public final boolean test(Map<Object, Object> ctx) throws Exception {
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_OFF_WAL);
+
+ try (Connection uploadConn = uploadConnection()) {
+ upload(uploadConn);
+ }
+
+ if (args.upload.disableWal())
+ executeUpdate(QueryFactory.TURN_ON_WAL);
+
+ return true;
+ }
+
+ /**
+ * Drops and re-creates test table.
+ */
+ private final void dropAndCreate() throws SQLException {
+ executeUpdate(QueryFactory.DROP_TABLE_IF_EXISTS);
+ executeUpdate(queries.createTable());
+ }
+
+ /**
+ * Retrieves records count in the test table.
+ */
+ public long count() throws SQLException {
+ try(PreparedStatement cnt = conn.get().prepareStatement(QueryFactory.COUNT)){
+ try (ResultSet rs = cnt.executeQuery()) {
+ rs.next();
+
+ return rs.getLong(1);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void tearDown() throws Exception {
+ BenchmarkUtils.println(cfg, "Tearing down");
+
+ try {
+ long cnt = count();
+
+ if (cnt != insertRowsCnt) {
+ String msg = "Rows count is incorrect: [actual=" + cnt + ", expected=" + insertRowsCnt + "]";
+
+ BenchmarkUtils.println(cfg, "TearDown: " + msg);
+
+ throw new RuntimeException(msg);
+ }
+
+ BenchmarkUtils.println(cfg, "Test table contains " + cnt + " rows.");
+ }
+ finally {
+ super.tearDown();
+ }
+
+ BenchmarkUtils.println(cfg, "TearDown successfully finished.");
+ }
+
+ /**
+ * Facility method for executing update queries.
+ */
+ private int executeUpdate(String updQry) throws SQLException {
+ try(PreparedStatement update = conn.get().prepareStatement(updQry)){
+ return update.executeUpdate();
+ }
+ }
+
+ /**
+ * Creates new connection only for upload purpose.
+ * This connection is special, since it may have additional jdbc url parameters.
+ */
+ private Connection uploadConnection() throws SQLException {
+ String urlParams = "";
+
+ // We can't just pass entire params string, due to yardstick, which relies on bash,
+ // has some troubles with escaping ampersand character.
+ List<String> rawParams = args.upload.uploadJdbcParams();
+
+ if (!rawParams.isEmpty()) {
+ String kvList = String.join("&", rawParams);
+ urlParams = "?" + kvList;
+ }
+
+ return connection(url + urlParams);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
new file mode 100644
index 0000000..a460e20
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/BatchedInsertBenchmark.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of data upload using batched insert.
+ * Supports streaming.
+ */
+public class BatchedInsertBenchmark extends AbstractUploadBenchmark {
+ /** Number of inserts in batch */
+ private long batchSize;
+
+ /** {@inheritDoc} */
+ @Override protected void init() {
+ batchSize = args.upload.jdbcBatchSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void warmup(Connection warmupConn) throws SQLException {
+ performBatchUpdate(warmupConn, warmupRowsCnt);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void upload(Connection uploadConn) throws Exception {
+ performBatchUpdate(uploadConn, insertRowsCnt);
+ }
+
+ /**
+ * Actually performs batched inserts, using specified connection.
+ *
+ * @param uploadConn Special connection for upload purposes.
+ * @param rowsCnt Number of rows to insert.
+ */
+ private void performBatchUpdate(Connection uploadConn, long rowsCnt) throws SQLException {
+ try (PreparedStatement insert = uploadConn.prepareStatement(queries.insert())) {
+ for (long id = 1; id <= rowsCnt; id++) {
+ queries.setRandomInsertArgs(insert, id);
+
+ insert.addBatch();
+
+ if (id % batchSize == 0 || id == rowsCnt)
+ insert.executeBatch();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
new file mode 100644
index 0000000..f0dcf20
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/CopyBenchmark.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import org.yardstickframework.BenchmarkUtils;
+
+/**
+ * Single shot benchmark for the COPY FROM... sql query.
+ * Measures total time of load number of rows from csv file using thin driver.
+ *
+ * Enabling streaming is useless for this benchmark, since copy uses streamer under the hood.
+ */
+public class CopyBenchmark extends AbstractUploadBenchmark {
+ /** csv file for warmup. */
+ private String warmupCsvPath;
+
+ /** csv file for benchmared action. */
+ private String realCsvPath;
+
+ /** packet_size parameter for sql COPY command.*/
+ private Long packetSize;
+
+ /** {@inheritDoc} */
+ @Override protected void init() {
+ super.init();
+
+ packetSize = args.upload.copyPacketSize();
+
+ warmupCsvPath = generateWarmupCsv();
+ realCsvPath = generateRealCsv();
+
+ }
+
+ /**
+ * Generate csv file for copy operation being benchmarked.
+ */
+ private String generateRealCsv() {
+ String prefix = "data-" + insertRowsCnt + "-rows-";
+
+ return generate(prefix, insertRowsCnt);
+ }
+
+ /**
+ * Generate csv file for copy operation being performed during warmup.
+ */
+ private String generateWarmupCsv() {
+ String prefix = "warmup-" + warmupRowsCnt + "-rows-" ;
+
+ return generate(prefix, warmupRowsCnt);
+ }
+
+ /**
+ * Generate csv file with specified prefix and containing specified number
+ * of records (lines).
+ * <br/>
+ * Each record contain incremental (starting 1) id and set of random values.
+ * <br/>
+ * File is created in current directory and is deleted on JVM exits.
+ *
+ * @param filePrefix prefix for csv to be generated.
+ * @param records how many csv lines generate.
+ *
+ * @return absolute path for generated csv file.
+ */
+ private String generate(String filePrefix, long records){
+ try {
+ File workDir = new File(System.getProperty("user.dir"));
+
+ File f = File.createTempFile(filePrefix, ".csv", workDir);
+
+ f.deleteOnExit();
+
+ BenchmarkUtils.println("Generating file: " + f.getAbsolutePath());
+
+ try (PrintStream out = new PrintStream(f)) {
+ for (long id = 1; id <= records; id++) {
+ String csvLine = queries.randomCsvLine(id);
+ out.println(csvLine);
+ }
+ }
+
+ long sizeMb = f.length() / (1024L * 1024L);
+
+ BenchmarkUtils.println("File have been generated (" + sizeMb + "MiB). It will be removed on exit");
+
+ return f.getAbsolutePath();
+ }
+ catch (Exception ex){
+ throw new RuntimeException("Couldn't generate CSV file, terminating", ex);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void warmup(Connection warmupConn) throws Exception {
+ try (PreparedStatement fromCsv = warmupConn.prepareStatement(queries.copyFrom(warmupCsvPath, packetSize))) {
+ fromCsv.executeUpdate();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void upload(Connection uploadConn) throws Exception {
+ try (PreparedStatement fromCsv = uploadConn.prepareStatement(queries.copyFrom(realCsvPath, packetSize))) {
+ fromCsv.executeUpdate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
new file mode 100644
index 0000000..ecf5412
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/InsertBenchmark.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Measures total time of upload data using sequence of single inserts;
+ * Supports streaming.
+ */
+public class InsertBenchmark extends AbstractUploadBenchmark {
+ /** {@inheritDoc} */
+ @Override public void warmup(Connection warmupConn) throws SQLException {
+ try (PreparedStatement insert = warmupConn.prepareStatement(queries.insert())) {
+ for (long id = 1; id <= warmupRowsCnt; id++) {
+ queries.setRandomInsertArgs(insert, id);
+
+ insert.executeUpdate();
+ }
+ }
+ }
+
+ /**
+ * Sequence of single inserts.
+ */
+ @Override public void upload (Connection uploadConn) throws Exception {
+ try (PreparedStatement insert = uploadConn.prepareStatement(queries.insert())) {
+ for (long id = 1; id <= insertRowsCnt; id++) {
+ queries.setRandomInsertArgs(insert, id);
+
+ insert.executeUpdate();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
new file mode 100644
index 0000000..60fbbda
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativePutBenchmark.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that inserts single upload of number of entries using {@link IgniteCache#put(Object, Object)}.
+ */
+public class NativePutBenchmark extends AbstractNativeBenchmark {
+ /**
+ * Uploads randomly generated data using simple put.
+ *
+ * @param cacheName - name of the cache.
+ * @param insertsCnt - how many entries should be uploaded.
+ */
+ @Override protected void upload(String cacheName, long insertsCnt) {
+ IgniteCache<Object, Object> c = ignite().cache(cacheName);
+
+ for (long id = 1; id <= insertsCnt; id++)
+ c.put(id, new Values10() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
new file mode 100644
index 0000000..e5117bc
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/NativeStreamerBenchmark.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.yardstick.upload.model.Values10;
+
+/**
+ * Benchmark that performs single upload of number of entries using {@link IgniteDataStreamer}.
+ */
+public class NativeStreamerBenchmark extends AbstractNativeBenchmark {
+ /**
+ * Uploads randomly generated entries to specified cache.
+ *
+ * @param cacheName - name of the cache.
+ * @param insertsCnt - how many entries should be uploaded.
+ */
+ @Override protected void upload(String cacheName, long insertsCnt) {
+ try (IgniteDataStreamer<Long, Values10> streamer = ignite().dataStreamer(cacheName)) {
+ if (args.upload.streamerBufSize() != null)
+ streamer.perNodeBufferSize(args.upload.streamerBufSize());
+
+ if (args.upload.streamerNodeParOps() != null)
+ streamer.perNodeParallelOperations(args.upload.streamerNodeParOps());
+
+ int batchSize = args.upload.streamerLocBatchSize();
+
+ // IgniteDataStreamer.addData(Object, Object) has known performance issue,
+ // so we have an option to work it around.
+ if (batchSize == 1) {
+ for (long i = 1; i <= insertsCnt; i++)
+ streamer.addData(i, new Values10());
+ }
+ else {
+ Map<Long, Values10> buf = new HashMap<>(batchSize);
+
+ for (long i = 1; i <= insertsCnt; i++) {
+ buf.put(i, new Values10());
+
+ if (i % batchSize == 0 || i == insertsCnt) {
+ streamer.addData(buf);
+
+ buf.clear();
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
new file mode 100644
index 0000000..27a2164
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/UploadBenchmarkArguments.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload;
+
+import com.beust.jcommander.Parameter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
+import org.apache.ignite.yardstick.IgniteBenchmarkArguments;
+
+/**
+ * Represents command line arguments that are specific to upload benchmarks.
+ *
+ * @see IgniteBenchmarkArguments
+ */
+@SuppressWarnings({"UnusedDeclaration", "FieldCanBeLocal"})
+public class UploadBenchmarkArguments {
+ /** Whither or not temporary disable Write Ahead Log during upload. */
+ @Parameter(names = {"--disable-wal"},
+ arity = 1,
+ description = "Upload benchmark only: " +
+ "turn off Write Ahead Log before data uploading " +
+ "and turn it on again when upload is done.")
+ private boolean disableWal = false;
+
+ /**
+ * Parameters for JDBC connection, that only uploads data.
+ *
+ * We can't just pass entire params string, due to yardstick, which relies on bash,
+ * has some troubles with escaping ampersand character.
+ */
+ @Parameter(names = {"--sql-jdbc-params"},
+ variableArity = true,
+ description = "Upload benchmark only: " +
+ "Additional url parameters (space separated key=value) for special JDBC connection that only uploads data. ")
+ private List<String> uploadJdbcParams = Collections.emptyList();
+
+ @Parameter(names = {"--sql-copy-packet-size"},
+ description = "Upload benchmark only: use custom packet_size (in bytes) for copy command.")
+ private Long copyPacketSize = null;
+
+ @Parameter(names = {"--streamer-node-buf-size"},
+ description = "Native Streamer benchmark only: Set streamer's perNodeBufferSize property")
+ private Integer streamerBufSize = null;
+
+ @Parameter(names = {"--streamer-node-par-ops"},
+ description = "Native Streamer benchmark only: Set streamer's perNodeParallelOperations property")
+ private Integer streamerNodeParOps = null;
+
+ @Parameter(names = {"--streamer-local-batch-size"},
+ description = "Native Streamer benchmark only: collect entries to local map before passing to streamer." +
+ "If set to 1, than entries will be passed directly.")
+ private int streamerLocBatchSize = 1;
+
+ /** How many rows to upload during warmup. */
+ @Parameter(names = {"--upload-warmup-rows"})
+ private long warmupRowsCnt = 3_000_000;
+
+ /** How many rows to upload during real test. */
+ @Parameter(names = {"--upload-rows"})
+ private long uploadRowsCnt = -1;
+
+ /** How many rows to include in each batch ({@link BatchedInsertBenchmark} only). */
+ @Parameter(names = {"--upload-jdbc-batch-size"})
+ private long jdbcBatchSize = -1;
+
+ /**
+ * @return Switch wal.
+ */
+ public boolean disableWal() {
+ return disableWal;
+ }
+
+ /** @return parameters for JDBC url. */
+ public List<String> uploadJdbcParams() {
+ return uploadJdbcParams;
+ }
+
+ /** @return packet_size value for copy command or {@code null} for default value. */
+ @Nullable
+ public Long copyPacketSize() {
+ return copyPacketSize;
+ }
+
+ /**
+ * @return Value for {@link IgniteDataStreamer#perNodeBufferSize(int)}.
+ */
+ @Nullable
+ public Integer streamerBufSize() {
+ return streamerBufSize;
+ }
+
+ /**
+ * @return Value for {@link IgniteDataStreamer#perNodeParallelOperations(int)}.
+ */
+ @Nullable
+ public Integer streamerNodeParOps() {
+ return streamerNodeParOps;
+ }
+
+ /**
+ * How many entries to collect before passing map to {@link IgniteDataStreamer#addData(Map)}.
+ * If set to 1, {@link IgniteDataStreamer#addData(Object, Object)} method will be used.
+ */
+ public int streamerLocBatchSize() {
+ return streamerLocBatchSize;
+ }
+
+ /**
+ * See {@link #warmupRowsCnt}.
+ */
+ public long warmupRowsCnt() {
+ return warmupRowsCnt;
+ }
+
+ /**
+ * See {@link #uploadRowsCnt}.
+ */
+ public long uploadRowsCnt() {
+ if (uploadRowsCnt < 0)
+ throw new IllegalStateException("Upload rows count is not specified. Check arguments.");
+
+ return uploadRowsCnt;
+ }
+
+ /**
+ * See {@link #jdbcBatchSize}.
+ */
+ public long jdbcBatchSize() {
+ if (jdbcBatchSize < 0)
+ throw new IllegalStateException("JDBC batch size is not specified. Check arguments.");
+
+ return jdbcBatchSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return GridToStringBuilder.toString(UploadBenchmarkArguments.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
new file mode 100644
index 0000000..e14a091
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/QueryFactory.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.concurrent.ThreadLocalRandom;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Factory that hides all test data details:
+ * what query to use to create table
+ * or what random arguments to set in prepared statement.
+ */
+public class QueryFactory {
+ /** Query to drop table if it exists. */
+ public static final String DROP_TABLE_IF_EXISTS = "DROP TABLE IF EXISTS test_upload;";
+
+ /** Query to count table size. */
+ public static final String COUNT = "SELECT COUNT(id) FROM test_upload;";
+
+ /** Turns off Write Ahead Log. */
+ public static final String TURN_OFF_WAL = "ALTER TABLE test_upload NOLOGGING";
+
+ /** Turns on Write Ahead Log. */
+ public static final String TURN_ON_WAL = "ALTER TABLE test_upload LOGGING";
+
+ /** Number of "values" fields in the test table (any field except primary key). */
+ private int valFieldsCnt = 10;
+
+ /** Create table with long primary key and number of long and varchar fields */
+ private String createTable = newCreateTableQuery();
+
+ /** Parametrised query to insert new row. */
+ private String insert = newInsertQuery();
+
+ /**
+ * See {@link #createTable}.
+ */
+ private String newCreateTableQuery() {
+ StringBuilder create = new StringBuilder("CREATE TABLE test_upload (id LONG PRIMARY KEY");
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++) {
+ create.append(", val_").append(vi);
+
+ if (vi % 2 == 1)
+ create.append(" VARCHAR(255)");
+ else
+ create.append(" LONG");
+
+ }
+
+ create.append(");");
+
+ return create.toString();
+ }
+
+ /**
+ * See {@link #insert}.
+ */
+ private String newInsertQuery() {
+ StringBuilder insert = new StringBuilder("INSERT INTO test_upload VALUES (?");
+ for (int vi = 1; vi <= valFieldsCnt; vi++)
+ insert.append(", ?");
+
+ insert.append(");");
+ return insert.toString();
+ }
+
+ /**
+ * See {@link #createTable}.
+ */
+ public String createTable() {
+ return createTable;
+ }
+
+ /**
+ * See {@link #insert}.
+ */
+ public String insert() {
+ return insert;
+ }
+
+ /**
+ * @param csvFilePath path to csv file.
+ * @param packetSize if not null, add packet_size query option.
+ * @return sql query that inserts data from specified csv file.
+ */
+ public String copyFrom(String csvFilePath, @Nullable Long packetSize) {
+ String pSizeExpr = "";
+
+ if (packetSize != null)
+ pSizeExpr = " packet_size " + packetSize;
+
+ return "COPY FROM \"" + csvFilePath + "\" " +
+ "INTO test_upload " + attributes() + " " +
+ "FORMAT CSV" + pSizeExpr + ";";
+ }
+
+ /**
+ * Creates string - comma-separated attributes of test table, surrounded with braces
+ * Is used as a part of sql statement.
+ *
+ * @return attributes list of test table as part of sql statement.
+ */
+ private String attributes() {
+ StringBuilder attrs = new StringBuilder("(id");
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++)
+ attrs.append(", val_").append(vi);
+
+ attrs.append(')');
+
+ return attrs.toString();
+ }
+
+ /**
+ * Fills specified prepared statement with random values and specified id (primary key).
+ *
+ * @param stmt prepared statement, built from {@link #insert} query.
+ * @param id id in the test table.
+ * @throws SQLException if statement is not correct.
+ */
+ public void setRandomInsertArgs(PreparedStatement stmt, long id) throws SQLException {
+ stmt.setLong(1, id);
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++) {
+ // vi is value index (among all values), but we also have "id" which is primary key
+ // so index in query is value index shifted by 1
+ int qryIdx = vi + 1;
+
+ long nextVal = rnd.nextLong();
+
+ if (vi % 2 == 1)
+ stmt.setLong(qryIdx, nextVal);
+ else
+ stmt.setString(qryIdx, String.valueOf(nextVal));
+ }
+ }
+
+ /**
+ * Generates CSV line containing specified id and random values.
+ * This line corresponds 1 row of the test table,
+ * which will be inserted in the end.
+ *
+ * @param id key in the test table.
+ * @return generated comma-separated line.
+ */
+ public String randomCsvLine(long id) {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ StringBuilder line = new StringBuilder().append(id);
+
+ for (int vi = 1; vi <= valFieldsCnt; vi++) {
+ line.append(',');
+
+ if (vi % 2 == 1)
+ line.append(rnd.nextLong());
+ else
+ line.append('"').append(rnd.nextLong()).append('"');
+ }
+
+ return line.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
new file mode 100644
index 0000000..f0bdee0
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/upload/model/Values10.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.upload.model;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Describes data model.
+ * Matches data model, defined in {@link QueryFactory#createTable()}
+ */
+public class Values10 {
+ /** */
+ final String val1;
+
+ /** */
+ final long val2;
+
+ /** */
+ final String val3;
+
+ /** */
+ final long val4;
+
+ /** */
+ final String val5;
+
+ /** */
+ final long val6;
+
+ /** */
+ final String val7;
+
+ /** */
+ final long val8;
+
+ /** */
+ final String val9;
+
+ /** */
+ final long val10;
+
+ /** Creates new object with randomly initialized fields */
+ public Values10(){
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ val1 = String.valueOf(rnd.nextLong());
+ val2 = rnd.nextLong();
+
+ val3 = String.valueOf(rnd.nextLong());
+ val4 = rnd.nextLong();
+
+ val5 = String.valueOf(rnd.nextLong());
+ val6 = rnd.nextLong();
+
+ val7 = String.valueOf(rnd.nextLong());
+ val8 = rnd.nextLong();
+
+ val9 = String.valueOf(rnd.nextLong());
+ val10 = rnd.nextLong();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/yardstickframework/package-info.java b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
new file mode 100644
index 0000000..989a667
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/yardstickframework/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Additional classes for yardstick framework
+ * This package name chosen so, according to default package, yardstick
+ * searches for classes to load
+ */
+
+package org.yardstickframework;
+
http://git-wip-us.apache.org/repos/asf/ignite/blob/ac3d5450/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
new file mode 100644
index 0000000..e4153ab
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/yardstickframework/probes/TotalTimeProbe.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.yardstickframework.probes;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import org.yardstickframework.BenchmarkConfiguration;
+import org.yardstickframework.BenchmarkDriver;
+import org.yardstickframework.BenchmarkExecutionAwareProbe;
+import org.yardstickframework.BenchmarkProbePoint;
+import org.yardstickframework.BenchmarkTotalsOnlyProbe;
+
+import static org.yardstickframework.BenchmarkUtils.println;
+
+/**
+ * Probe that measures total execution time of action for each thread.
+ */
+public class TotalTimeProbe implements BenchmarkExecutionAwareProbe, BenchmarkTotalsOnlyProbe {
+ /** Thread local agents. */
+ private volatile ThreadAgent[] agents;
+
+ /** Benchmark configuration. */
+ private BenchmarkConfiguration cfg;
+
+ /** {@inheritDoc} */
+ @Override public void beforeExecute(int threadIdx) {
+ agents[threadIdx].before();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void afterExecute(int threadIdx) {
+ agents[threadIdx].after();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start(BenchmarkDriver drv, BenchmarkConfiguration cfg) throws Exception {
+ this.cfg = cfg;
+
+ agents = new ThreadAgent[cfg.threads()];
+
+ for (int thId = 0; thId < agents.length; thId++)
+ agents[thId] = new ThreadAgent();
+
+ println(cfg, "TotalTime probe started");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws Exception {
+ println(cfg, "TotalTime probe stopped");
+ }
+
+ /**
+ * This probe producing two-dimension points:
+ * Number of thread that executed {@code test} method and
+ * time it took to finish.
+ *
+ * @return meta information about each point.
+ */
+ @Override public Collection<String> metaInfo() {
+ return Arrays.asList("Thread ID", "Duration, ms");
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<BenchmarkProbePoint> points() {
+ Collection<BenchmarkProbePoint> points = new ArrayList<>();
+
+ for (int thId = 0; thId < agents.length; thId++) {
+
+ double durationMs = agents[thId].getDurationNs() / 1000_000d;
+
+ BenchmarkProbePoint pnt = new BenchmarkProbePoint(thId, new double[] {durationMs});
+
+ points.add(pnt);
+ }
+
+ return points;
+ }
+
+ /**
+ * Useless for {@link BenchmarkTotalsOnlyProbe}.
+ */
+ @Override public void buildPoint(long time) {
+ // No-op
+ }
+
+ /**
+ * Measures execution time. Used in thread local context.
+ */
+ static class ThreadAgent {
+ /** Start time of {@link org.yardstickframework.BenchmarkDriver#test(java.util.Map)} method. */
+ private volatile long startTs;
+
+ /** Finish time. */
+ private volatile long finishTs;
+
+ /**
+ * Start measuring.
+ */
+ public void before(){
+ startTs = System.nanoTime();
+ }
+
+ /**
+ * Stop measuring.
+ */
+ public void after(){
+ finishTs = System.nanoTime();
+ }
+
+ /**
+ * @return measured duration in nanoseconds.
+ */
+ public long getDurationNs(){
+ if (finishTs == 0L && startTs != 0)
+ throw new IllegalStateException("Cannot provide results of TotalTime probe; " +
+ "after() method haven't been called");
+
+ return (finishTs - startTs);
+ }
+ }
+}