You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2018/11/12 08:30:12 UTC
ignite git commit: IGNITE-9712: Added SQL benchmarks for simple
SELECT statements. This closes #4866.
Repository: ignite
Updated Branches:
refs/heads/master 43b9cfa34 -> f8e15bb9b
IGNITE-9712: Added SQL benchmarks for simple SELECT statements. This closes #4866.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f8e15bb9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f8e15bb9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f8e15bb9
Branch: refs/heads/master
Commit: f8e15bb9bfd70ad50668c72cd76a848fd1f49279
Parents: 43b9cfa
Author: Pavel Kuznetsov <pa...@gmail.com>
Authored: Mon Nov 12 11:30:04 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Mon Nov 12 11:30:04 2018 +0300
----------------------------------------------------------------------
.../config/benchmark-select.properties | 159 +++++++++++++++++++
.../yardstick/IgniteBenchmarkArguments.java | 16 +-
.../jdbc/JdbcSqlQueryRangeBenchmark.java | 29 +++-
.../apache/ignite/yardstick/jdbc/JdbcUtils.java | 25 ++-
.../jdbc/NativeSqlQueryRangeBenchmark.java | 24 ++-
.../ignite/yardstick/jdbc/SelectCommand.java | 85 ++++++++++
6 files changed, 318 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8e15bb9/modules/yardstick/config/benchmark-select.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-select.properties b/modules/yardstick/config/benchmark-select.properties
new file mode 100644
index 0000000..1f85f58
--- /dev/null
+++ b/modules/yardstick/config/benchmark-select.properties
@@ -0,0 +1,159 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Benchmarks for SELECT operations. Operations are measured via JDBC thin, JDBC Thick and native sql.
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+-Xms8g \
+-Xmx8g \
+-Xloggc:./gc${now0}.log \
+-XX:+PrintGCDetails \
+-verbose:gc \
+-XX:+UseParNewGC \
+-XX:+UseConcMarkSweepGC \
+-XX:+PrintGCDateStamps \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers before every benchmark execution.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# The benchmark is applicable only for 2 servers (the second server is started in client mode) and 1 driver.
+SERVER_HOSTS=localhost,localhost
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=30
+
+# Duration.
+d=300
+
+# Threads count.
+t=4
+
+# Sync mode.
+sm=FULL_SYNC
+
+# Jobs.
+j=10
+
+
+# Note: don't forget to update --clientNodesAfterId value if benchmark is run on more than 2 hosts.
+commonProps="\
+ -cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml \
+ -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} \
+ -sm ${sm} -sn IgniteNode \
+ --clientNodesAfterId 0 \
+ -cl \
+ "
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
+CONFIGS="\
+${commonProps} -jdbc jdbc:ignite:thin://auto.find/ \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_PRIMARY_KEY \
+ --sqlRange 1 \
+ -ds ${ver}sql-select-pk-jdbc-thin-r1-${b}-backup, \
+${commonProps} -jdbc jdbc:ignite:cfg://cache=query@config/ignite-jdbc-config.xml \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_PRIMARY_KEY \
+ --sqlRange 1 \
+ -ds ${ver}sql-select-pk-jdbc-thick-r1-${b}-backup, \
+${commonProps} \
+ -dn NativeSqlQueryRangeBenchmark \
+ --select-command BY_PRIMARY_KEY \
+ --sqlRange 1 \
+ -ds ${ver}sql-select-pk-native-sql-r1-${b}-backup, \
+ \
+${commonProps} -jdbc jdbc:ignite:thin://auto.find/ \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_VALUE \
+ --sqlRange 1 \
+ -ds ${ver}sql-select-val-jdbc-thin-r1-${b}-backup, \
+${commonProps} -jdbc jdbc:ignite:cfg://cache=query@config/ignite-jdbc-config.xml \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_VALUE \
+ --sqlRange 1 \
+ -ds ${ver}sql-select-val-jdbc-thick-r1-${b}-backup, \
+${commonProps} \
+ -dn NativeSqlQueryRangeBenchmark \
+ --select-command BY_VALUE \
+ --sqlRange 1 \
+ -ds ${ver}sql-select-val-native-sql-r1-${b}-backup, \
+ \
+${commonProps} -jdbc jdbc:ignite:thin://auto.find/ \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_PRIMARY_KEY \
+ --sqlRange 1000 \
+ -ds ${ver}sql-select-pk-jdbc-thin-r1000-${b}-backup, \
+${commonProps} -jdbc jdbc:ignite:cfg://cache=query@config/ignite-jdbc-config.xml \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_PRIMARY_KEY \
+ --sqlRange 1000 \
+ -ds ${ver}sql-select-pk-jdbc-thick-r1000-${b}-backup, \
+${commonProps} \
+ -dn NativeSqlQueryRangeBenchmark \
+ --select-command BY_PRIMARY_KEY \
+ --sqlRange 1000 \
+ -ds ${ver}sql-select-pk-native-sql-r1000-${b}-backup, \
+ \
+${commonProps} -jdbc jdbc:ignite:thin://auto.find/ \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_VALUE \
+ --sqlRange 1000 \
+ -ds ${ver}sql-select-val-jdbc-thin-r1000-${b}-backup, \
+${commonProps} -jdbc jdbc:ignite:cfg://cache=query@config/ignite-jdbc-config.xml \
+ -dn JdbcSqlQueryRangeBenchmark \
+ --select-command BY_VALUE \
+ --sqlRange 1000 \
+ -ds ${ver}sql-select-val-jdbc-thick-r1000-${b}-backup, \
+${commonProps} \
+ -dn NativeSqlQueryRangeBenchmark \
+ --select-command BY_VALUE \
+ --sqlRange 1000 \
+ -ds ${ver}sql-select-val-native-sql-r1000-${b}-backup \
+"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8e15bb9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
index 3f4fddc..4f8b83b 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/IgniteBenchmarkArguments.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
import org.apache.ignite.yardstick.cache.IgniteStreamerBenchmark;
+import org.apache.ignite.yardstick.jdbc.SelectCommand;
import org.apache.ignite.yardstick.upload.UploadBenchmarkArguments;
import org.jetbrains.annotations.Nullable;
@@ -289,6 +290,10 @@ public class IgniteBenchmarkArguments {
@GridToStringInclude
public long mvccContentionRange = 10_000;
+ /** See {@link #selectCommand()}. */
+ @Parameter(names = {"--select-command"})
+ private SelectCommand selectCommand = SelectCommand.BY_PRIMARY_KEY;
+
/**
* @return {@code True} if need set {@link DataStorageConfiguration}.
*/
@@ -402,7 +407,7 @@ public class IgniteBenchmarkArguments {
}
/** With what cache atomicity mode to create tables. */
- @Nullable public CacheAtomicityMode atomicMode(){
+ @Nullable public CacheAtomicityMode atomicMode() {
return atomicMode;
}
@@ -712,6 +717,15 @@ public class IgniteBenchmarkArguments {
return mvccContentionRange;
}
+ /**
+ * @return What type of SQL SELECT queries to execute. It affects what type of field will present in the WHERE
+ * clause: PK, indexed value field, etc.
+ * @see SelectCommand
+ */
+ public SelectCommand selectCommand() {
+ return selectCommand;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return GridToStringBuilder.toString(IgniteBenchmarkArguments.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8e15bb9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcSqlQueryRangeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcSqlQueryRangeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcSqlQueryRangeBenchmark.java
index c1adb85..5babbf7 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcSqlQueryRangeBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcSqlQueryRangeBenchmark.java
@@ -21,18 +21,31 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
+import org.yardstickframework.BenchmarkConfiguration;
/**
* JDBC benchmark that performs select operations
*/
public class JdbcSqlQueryRangeBenchmark extends AbstractJdbcBenchmark {
/** Statement with range. */
- private ThreadLocal<PreparedStatement> stmtRange = newStatement(
- "SELECT id, val FROM test_long WHERE id BETWEEN ? AND ?");
+ private ThreadLocal<PreparedStatement> stmtRange;
/** Statement full scan. */
- private ThreadLocal<PreparedStatement> stmtSingle = newStatement(
- "SELECT id, val FROM test_long WHERE id = ?");
+ private ThreadLocal<PreparedStatement> stmtSingle;
+
+ /** Prepares SELECT query and it's parameters. */
+ private SelectCommand select;
+
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ select = args.selectCommand();
+
+ stmtRange = newStatement(select.selectRange());
+
+ stmtSingle = newStatement(select.selectOne());
+ }
/** {@inheritDoc} */
@Override public boolean test(Map<Object, Object> ctx) throws Exception {
@@ -43,7 +56,9 @@ public class JdbcSqlQueryRangeBenchmark extends AbstractJdbcBenchmark {
if (args.sqlRange() == 1) {
stmt = stmtSingle.get();
- stmt.setLong(1, ThreadLocalRandom.current().nextLong(args.range()) + 1);
+ long id = ThreadLocalRandom.current().nextLong(args.range()) + 1;
+
+ stmt.setLong(1, select.fieldByPK(id));
expRsSize = 1;
}
@@ -53,8 +68,8 @@ public class JdbcSqlQueryRangeBenchmark extends AbstractJdbcBenchmark {
long id = ThreadLocalRandom.current().nextLong(args.range() - args.sqlRange()) + 1;
long maxId = id + args.sqlRange() - 1;
- stmt.setLong(1, id);
- stmt.setLong(2, maxId);
+ stmt.setLong(1, select.fieldByPK(id));
+ stmt.setLong(2, select.fieldByPK(maxId));
expRsSize = args.sqlRange();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8e15bb9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcUtils.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcUtils.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcUtils.java
index 81d6c17..2b6c186 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcUtils.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/JdbcUtils.java
@@ -20,6 +20,7 @@ package org.apache.ignite.yardstick.jdbc;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
import org.yardstickframework.BenchmarkConfiguration;
import static org.yardstickframework.BenchmarkUtils.println;
@@ -37,26 +38,36 @@ public class JdbcUtils {
public static void fillData(BenchmarkConfiguration cfg, IgniteEx ignite, long range, CacheAtomicityMode atomicMode) {
println(cfg, "Create table...");
- String withExpr = atomicMode != null ? " WITH \"atomicity=" + atomicMode.name() + "\";" : ";";
+ StringBuilder qry = new StringBuilder("CREATE TABLE test_long (id LONG PRIMARY KEY, val LONG)" +
+ " WITH \"wrap_value=true");
- String qry = "CREATE TABLE test_long (id long primary key, val long)" + withExpr;
+ if (atomicMode != null)
+ qry.append(", atomicity=").append(atomicMode.name());
- println(cfg, "Creating table with schema: " + qry);
+ qry.append("\";");
- ignite.context().query().querySqlFields(
- new SqlFieldsQuery(qry), true);
+ String qryStr = qry.toString();
+
+ println(cfg, "Creating table with schema: " + qryStr);
+
+ GridQueryProcessor qProc = ignite.context().query();
+
+ qProc.querySqlFields(
+ new SqlFieldsQuery(qryStr), true);
println(cfg, "Populate data...");
for (long l = 1; l <= range; ++l) {
- ignite.context().query().querySqlFields(
- new SqlFieldsQuery("insert into test_long (id, val) values (?, ?)")
+ qProc.querySqlFields(
+ new SqlFieldsQuery("INSERT INTO test_long (id, val) VALUES (?, ?)")
.setArgs(l, l + 1), true);
if (l % 10000 == 0)
println(cfg, "Populate " + l);
}
+ qProc.querySqlFields(new SqlFieldsQuery("CREATE INDEX val_idx ON test_long (val)"), true);
+
println(cfg, "Finished populating data");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8e15bb9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/NativeSqlQueryRangeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/NativeSqlQueryRangeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/NativeSqlQueryRangeBenchmark.java
index 8dcdda7..a5dac27 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/NativeSqlQueryRangeBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/NativeSqlQueryRangeBenchmark.java
@@ -23,35 +23,49 @@ import java.util.concurrent.ThreadLocalRandom;
import org.apache.ignite.cache.query.FieldsQueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.internal.IgniteEx;
+import org.yardstickframework.BenchmarkConfiguration;
/**
* Native sql benchmark that performs select operations.
*/
public class NativeSqlQueryRangeBenchmark extends AbstractNativeBenchmark {
+ /** Prepares SELECT query and it's parameters. */
+ private SelectCommand select;
+
+ /** {@inheritDoc} */
+ @Override public void setUp(BenchmarkConfiguration cfg) throws Exception {
+ super.setUp(cfg);
+
+ select = args.selectCommand();
+ }
+
/**
* Benchmarked action that performs selects and validates results.
*
* {@inheritDoc}
*/
+
@Override public boolean test(Map<Object, Object> ctx) throws Exception {
long expRsSize;
SqlFieldsQuery qry;
if (args.sqlRange() == 1) {
- qry = new SqlFieldsQuery("SELECT id, val FROM test_long WHERE id = ?");
+ qry = new SqlFieldsQuery(select.selectOne());
+
+ long id = ThreadLocalRandom.current().nextLong(args.range()) + 1;
- qry.setArgs(ThreadLocalRandom.current().nextLong(args.range()) + 1);
+ qry.setArgs(select.fieldByPK(id));
expRsSize = 1;
}
else {
- qry = new SqlFieldsQuery("SELECT id, val FROM test_long WHERE id BETWEEN ? AND ?");
+ qry = new SqlFieldsQuery(select.selectRange());
long id = ThreadLocalRandom.current().nextLong(args.range() - args.sqlRange()) + 1;
long maxId = id + args.sqlRange() - 1;
- qry.setArgs(id, maxId);
+ qry.setArgs(select.fieldByPK(id), select.fieldByPK(maxId));
expRsSize = args.sqlRange();
}
@@ -59,7 +73,7 @@ public class NativeSqlQueryRangeBenchmark extends AbstractNativeBenchmark {
long rsSize = 0;
try (FieldsQueryCursor<List<?>> cursor = ((IgniteEx)ignite()).context().query()
- .querySqlFields(qry, false)) {
+ .querySqlFields(qry, false)) {
for (List<?> row : cursor) {
if ((Long)row.get(0) + 1 != (Long)row.get(1))
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8e15bb9/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/SelectCommand.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/SelectCommand.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/SelectCommand.java
new file mode 100644
index 0000000..468d073
--- /dev/null
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/jdbc/SelectCommand.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.jdbc;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.yardstickframework.BenchmarkConfiguration;
+
+/**
+ * Creates SELECT queries and arguments for those queries. Uses data model defined in {@link
+ * JdbcUtils#fillData(BenchmarkConfiguration, IgniteEx, long, CacheAtomicityMode)}
+ */
+public enum SelectCommand {
+ /**
+ * Creates SELECT query that has primary key field in the WHERE clause.
+ */
+ BY_PRIMARY_KEY {
+ @Override public String selectOne() {
+ return "SELECT id, val FROM test_long WHERE id = ?;";
+ }
+
+ @Override public String selectRange() {
+ return "SELECT id, val FROM test_long WHERE id BETWEEN ? AND ?;";
+ }
+
+ @Override public long fieldByPK(long pk) {
+ // This command uses PK value itself.
+ return pk;
+ }
+ },
+
+ /**
+ * Creates SELECT query that has value field in the WHERE clause.
+ */
+ BY_VALUE {
+ @Override public String selectOne() {
+ return "SELECT id, val FROM test_long WHERE val = ?;";
+ }
+
+ @Override public String selectRange() {
+ return "SELECT id, val FROM test_long WHERE val BETWEEN ? AND ?;";
+ }
+
+ @Override public long fieldByPK(long pk) {
+ // data model defines that value is generated as id (PK) field plus one.
+ return pk + 1;
+ }
+ };
+
+ /**
+ * Create SELECT query that returns one row and has one parameter - field value for WHERE clause.
+ */
+ public abstract String selectOne();
+
+ /**
+ * Create SELECT query that returns number of rows. This query has 2 parameters - min and max value for BETWEEN
+ * operator from WHERE clause.
+ */
+ public abstract String selectRange();
+
+ /**
+ * Gets field value by primary key. Field can be PK itself or val field depending on type of this Select.
+ * Implementation of this method is based on how {@link JdbcUtils#fillData(BenchmarkConfiguration, IgniteEx, long,
+ * CacheAtomicityMode)} generates data.
+ *
+ * @param pk primary key for what to compute field value.
+ * @return field value.
+ */
+ public abstract long fieldByPK(long pk);
+}