You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2019/05/28 12:47:08 UTC
[ignite] branch master updated: IGNITE-11756: SQL: implement a
table row count statistics for the local queries. This closes #6529.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 81e46aa IGNITE-11756: SQL: implement a table row count statistics for the local queries. This closes #6529.
81e46aa is described below
commit 81e46aa75dec2277c55c714dc2d3a2406e603735
Author: rkondakov <ko...@mail.ru>
AuthorDate: Tue May 28 15:45:38 2019 +0300
IGNITE-11756: SQL: implement a table row count statistics for the local queries. This closes #6529.
---
.../query/h2/opt/GridH2SpatialIndex.java | 5 +
.../internal/processors/query/h2/H2QueryInfo.java | 41 ++---
.../processors/query/h2/IgniteH2Indexing.java | 9 +-
.../query/h2/LongRunningQueryManager.java | 7 +-
.../internal/processors/query/h2/QueryParser.java | 187 ++++++++++----------
.../query/h2/database/H2PkHashIndex.java | 36 +++-
.../query/h2/database/H2TreeClientIndex.java | 9 +-
.../processors/query/h2/database/H2TreeIndex.java | 18 ++
.../query/h2/database/H2TreeIndexBase.java | 5 -
.../processors/query/h2/opt/GridH2IndexBase.java | 15 +-
.../processors/query/h2/opt/GridH2Table.java | 98 ++++++++++-
.../processors/query/h2/opt/QueryContext.java | 15 +-
.../processors/query/h2/opt/TableStatistics.java | 53 ++++++
.../query/h2/twostep/GridMapQueryExecutor.java | 3 +-
.../query/h2/twostep/GridReduceQueryExecutor.java | 3 +-
.../CacheMvccSelectForUpdateQueryBasicTest.java | 21 ++-
.../query/IgniteSqlSegmentedIndexSelfTest.java | 2 +
...ountTableStatisticsSurvivesNodeRestartTest.java | 100 +++++++++++
.../query/h2/RowCountTableStatisticsUsageTest.java | 194 +++++++++++++++++++++
.../query/h2/TableStatisticsAbstractTest.java | 165 ++++++++++++++++++
.../query/h2/sql/GridQueryParsingTest.java | 43 ++++-
.../IgniteBinaryCacheQueryTestSuite.java | 6 +
22 files changed, 877 insertions(+), 158 deletions(-)
diff --git a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
index efd7899..6dca75b 100644
--- a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
+++ b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
@@ -416,6 +416,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
}
/** {@inheritDoc} */
+ @Override public long totalRowCount(IndexingQueryCacheFilter partsFilter) {
+ return rowCnt;
+ }
+
+ /** {@inheritDoc} */
@Override public Cursor findByGeometry(TableFilter filter, SearchRow first, SearchRow last,
SearchRow intersection) {
Lock l = lock.readLock();
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java
index 32ce756..0961929 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2QueryInfo.java
@@ -17,15 +17,15 @@
package org.apache.ignite.internal.processors.query.h2;
-import java.sql.Connection;
import java.sql.PreparedStatement;
-import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.h2.command.Prepared;
import org.h2.engine.Session;
/**
@@ -53,6 +53,9 @@ public class H2QueryInfo {
/** Lazy mode. */
private final boolean lazy;
+ /** Prepared statement. */
+ private final Prepared stmt;
+
/**
* @param type Query type.
* @param stmt Query statement.
@@ -74,6 +77,7 @@ public class H2QueryInfo {
enforceJoinOrder = s.isForceJoinOrder();
distributedJoin = s.isJoinBatchEnabled();
lazy = s.isLazyQueryExecution();
+ this.stmt = GridSqlQueryParser.prepared(stmt);
}
catch (SQLException e) {
throw new IgniteSQLException("Cannot collect query info", IgniteQueryErrorCode.UNKNOWN, e);
@@ -99,16 +103,16 @@ public class H2QueryInfo {
/**
* @param log Logger.
* @param msg Log message
- * @param connMgr Connection manager.
*/
- public void printLogMessage(IgniteLogger log, ConnectionManager connMgr, String msg) {
+ public void printLogMessage(IgniteLogger log, String msg) {
StringBuilder msgSb = new StringBuilder(msg + " [");
msgSb.append("time=").append(time()).append("ms")
.append(", type=").append(type)
.append(", distributedJoin=").append(distributedJoin)
.append(", enforceJoinOrder=").append(enforceJoinOrder)
- .append(", lazy=").append(lazy);
+ .append(", lazy=").append(lazy)
+ .append(", schema=").append(schema);
printInfo(msgSb);
@@ -116,7 +120,7 @@ public class H2QueryInfo {
.append(sql);
if (type != QueryType.REDUCE)
- msgSb.append("', plan=").append(queryPlan(log, connMgr));
+ msgSb.append("', plan=").append(stmt.getPlanSQL());
msgSb.append(']');
@@ -124,31 +128,6 @@ public class H2QueryInfo {
}
/**
- * @param log Logger.
- * @param connMgr Connection manager.
- * @return Query plan.
- */
- protected String queryPlan(IgniteLogger log, ConnectionManager connMgr) {
- Connection c = connMgr.connectionForThread().connection(schema);
-
- H2Utils.setupConnection(c, distributedJoin, enforceJoinOrder);
-
- try (PreparedStatement pstmt = c.prepareStatement("EXPLAIN " + sql)) {
-
- try (ResultSet plan = pstmt.executeQuery()) {
- plan.next();
-
- return plan.getString(1) + U.nl();
- }
- }
- catch (Exception e) {
- log.warning("Cannot get plan for long query: " + sql, e);
-
- return "[error on calculate plan: " + e.getMessage() + ']';
- }
- }
-
- /**
* Query type.
*/
public enum QueryType {
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 03117b8..b4238c2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -524,7 +524,8 @@ public class IgniteH2Indexing implements GridQueryIndexing {
filter,
null,
mvccSnapshot,
- null
+ null,
+ true
);
return new GridQueryFieldsResultAdapter(select.meta(), null) {
@@ -883,11 +884,13 @@ public class IgniteH2Indexing implements GridQueryIndexing {
ResultSet rs = executeSqlQuery(conn, stmt, timeoutMillis, cancel);
if (qryInfo != null && qryInfo.time() > longRunningQryMgr.getTimeout())
- qryInfo.printLogMessage(log, connMgr, "Long running query is finished"); return rs;
+ qryInfo.printLogMessage(log, "Long running query is finished");
+
+ return rs;
}
catch (Throwable e) {
if (qryInfo != null && qryInfo.time() > longRunningQryMgr.getTimeout()) {
- qryInfo.printLogMessage(log, connMgr, "Long running query is finished with error: "
+ qryInfo.printLogMessage(log, "Long running query is finished with error: "
+ e.getMessage());
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/LongRunningQueryManager.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/LongRunningQueryManager.java
index 24b078f..efc10d3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/LongRunningQueryManager.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/LongRunningQueryManager.java
@@ -32,9 +32,6 @@ public final class LongRunningQueryManager {
/** Check period in ms. */
private static final long CHECK_PERIOD = 1_000;
- /** Connection manager. */
- private final ConnectionManager connMgr;
-
/** Queries collection. Sorted collection isn't used to reduce 'put' time. */
private final ConcurrentHashMap<H2QueryInfo, TimeoutChecker> qrys = new ConcurrentHashMap<>();
@@ -62,8 +59,6 @@ public final class LongRunningQueryManager {
* @param ctx Kernal context.
*/
public LongRunningQueryManager(GridKernalContext ctx) {
- connMgr = ((IgniteH2Indexing)ctx.query().getIndexing()).connections();
-
log = ctx.log(LongRunningQueryManager.class);
checkWorker = new GridWorker(ctx.igniteInstanceName(), "long-qry", log) {
@@ -122,7 +117,7 @@ public final class LongRunningQueryManager {
H2QueryInfo qinfo = e.getKey();
if (e.getValue().checkTimeout(qinfo.time())) {
- qinfo.printLogMessage(log, connMgr, "Query execution is too long");
+ qinfo.printLogMessage(log, "Query execution is too long");
if (e.getValue().timeoutMult <= 1)
qrys.remove(qinfo);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryParser.java
index 0bb6922..33971ef 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/QueryParser.java
@@ -44,6 +44,8 @@ import org.apache.ignite.internal.processors.query.h2.dml.DmlAstUtils;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
+import org.apache.ignite.internal.processors.query.h2.opt.QueryContext;
+import org.apache.ignite.internal.processors.query.h2.opt.QueryContextRegistry;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlInsert;
@@ -280,17 +282,24 @@ public class QueryParser {
H2Utils.setupConnection(c, /*distributedJoins*/false, /*enforceJoinOrder*/enforceJoinOrderOnParsing);
- PreparedStatement stmt;
+ QueryContext qctx = new QueryContext(
+ 0,
+ idx.backupFilter(null, null),
+ null,
+ null,
+ null,
+ qry.isLocal()
+ );
+
+ QueryContextRegistry qryCtxRegistry = idx.queryContextRegistry();
+
+ qryCtxRegistry.setThreadLocal(qctx);
+
+ PreparedStatement stmt = null;
try {
stmt = connMgr.prepareStatementNoCache(c, qry.getSql());
- }
- catch (SQLException e) {
- throw new IgniteSQLException("Failed to parse query. " + e.getMessage(),
- IgniteQueryErrorCode.PARSING, e);
- }
- try {
if (qry.isLocal() && GridSqlQueryParser.checkMultipleStatements(stmt))
throw new IgniteSQLException("Multiple statements queries are not supported for local queries.",
IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
@@ -432,68 +441,50 @@ public class QueryParser {
// node stripes in parallel and then merged through reduce process.
boolean splitNeeded = !loc || locSplit;
- try {
- String forUpdateQryOutTx = null;
- String forUpdateQryTx = null;
- GridCacheTwoStepQuery forUpdateTwoStepQry = null;
-
- boolean forUpdate = GridSqlQueryParser.isForUpdateQuery(prepared);
-
- // SELECT FOR UPDATE case handling. We need to create extra queries with appended _key
- // column to be able to lock selected rows further.
- if (forUpdate) {
- // We have checked above that it's not an UNION query, so it's got to be SELECT.
- assert selectStmt instanceof GridSqlSelect;
-
- // Check FOR UPDATE invariants: only one table, MVCC is there.
- if (cacheIds.size() != 1)
- throw new IgniteSQLException("SELECT FOR UPDATE is supported only for queries " +
- "that involve single transactional cache.");
-
- if (mvccCacheId == null)
- throw new IgniteSQLException("SELECT FOR UPDATE query requires transactional cache " +
- "with MVCC enabled.", IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
-
- // We need a copy because we are going to modify AST a bit. We do not want to modify original select.
- GridSqlSelect selForUpdate = ((GridSqlSelect)selectStmt).copySelectForUpdate();
-
- // Clear forUpdate flag to run it as a plain query.
- selForUpdate.forUpdate(false);
- ((GridSqlSelect)selectStmt).forUpdate(false);
-
- // Remember sql string without FOR UPDATE clause.
- forUpdateQryOutTx = selForUpdate.getSQL();
-
- GridSqlAlias keyCol = keyColumn(selForUpdate);
-
- selForUpdate.addColumn(keyCol, true);
-
- // Remember sql string without FOR UPDATE clause and with _key column.
- forUpdateQryTx = selForUpdate.getSQL();
-
- // Prepare additional two-step query for FOR UPDATE case.
- if (splitNeeded) {
- forUpdateTwoStepQry = GridSqlQuerySplitter.split(
- connMgr.connectionForThread().connection(newQry.getSchema()),
- selForUpdate,
- forUpdateQryTx,
- newQry.isCollocated(),
- newQry.isDistributedJoins(),
- newQry.isEnforceJoinOrder(),
- locSplit,
- idx,
- paramsCnt
- );
- }
- }
+ String forUpdateQryOutTx = null;
+ String forUpdateQryTx = null;
+ GridCacheTwoStepQuery forUpdateTwoStepQry = null;
+
+ boolean forUpdate = GridSqlQueryParser.isForUpdateQuery(prepared);
- GridCacheTwoStepQuery twoStepQry = null;
+ // SELECT FOR UPDATE case handling. We need to create extra queries with appended _key
+ // column to be able to lock selected rows further.
+ if (forUpdate) {
+ // We have checked above that it's not an UNION query, so it's got to be SELECT.
+ assert selectStmt instanceof GridSqlSelect;
+ // Check FOR UPDATE invariants: only one table, MVCC is there.
+ if (cacheIds.size() != 1)
+ throw new IgniteSQLException("SELECT FOR UPDATE is supported only for queries " +
+ "that involve single transactional cache.");
+
+ if (mvccCacheId == null)
+ throw new IgniteSQLException("SELECT FOR UPDATE query requires transactional cache " +
+ "with MVCC enabled.", IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+ // We need a copy because we are going to modify AST a bit. We do not want to modify original select.
+ GridSqlSelect selForUpdate = ((GridSqlSelect)selectStmt).copySelectForUpdate();
+
+ // Clear forUpdate flag to run it as a plain query.
+ selForUpdate.forUpdate(false);
+ ((GridSqlSelect)selectStmt).forUpdate(false);
+
+ // Remember sql string without FOR UPDATE clause.
+ forUpdateQryOutTx = selForUpdate.getSQL();
+
+ GridSqlAlias keyCol = keyColumn(selForUpdate);
+
+ selForUpdate.addColumn(keyCol, true);
+
+ // Remember sql string without FOR UPDATE clause and with _key column.
+ forUpdateQryTx = selForUpdate.getSQL();
+
+ // Prepare additional two-step query for FOR UPDATE case.
if (splitNeeded) {
- twoStepQry = GridSqlQuerySplitter.split(
+ forUpdateTwoStepQry = GridSqlQuerySplitter.split(
connMgr.connectionForThread().connection(newQry.getSchema()),
- selectStmt,
- newQry.getSql(),
+ selForUpdate,
+ forUpdateQryTx,
newQry.isCollocated(),
newQry.isDistributedJoins(),
newQry.isEnforceJoinOrder(),
@@ -502,39 +493,53 @@ public class QueryParser {
paramsCnt
);
}
+ }
- List<GridQueryFieldMetadata> meta = H2Utils.meta(stmt.getMetaData());
+ GridCacheTwoStepQuery twoStepQry = null;
- QueryParserResultSelect select = new QueryParserResultSelect(
+ if (splitNeeded) {
+ twoStepQry = GridSqlQuerySplitter.split(
+ connMgr.connectionForThread().connection(newQry.getSchema()),
selectStmt,
- twoStepQry,
- forUpdateTwoStepQry,
- meta,
- cacheIds,
- mvccCacheId,
- forUpdateQryOutTx,
- forUpdateQryTx
+ newQry.getSql(),
+ newQry.isCollocated(),
+ newQry.isDistributedJoins(),
+ newQry.isEnforceJoinOrder(),
+ locSplit,
+ idx,
+ paramsCnt
);
-
- return new QueryParserResult(
- newQryDesc,
- QueryParameters.fromQuery(newQry),
- remainingQry,
- paramsMeta,
- select,
- null,
- null
- );
- }
- catch (IgniteCheckedException e) {
- throw new IgniteSQLException("Failed to parse query: " + newQry.getSql(), IgniteQueryErrorCode.PARSING,
- e);
- }
- catch (SQLException e) {
- throw new IgniteSQLException(e);
}
+
+ List<GridQueryFieldMetadata> meta = H2Utils.meta(stmt.getMetaData());
+
+ QueryParserResultSelect select = new QueryParserResultSelect(
+ selectStmt,
+ twoStepQry,
+ forUpdateTwoStepQry,
+ meta,
+ cacheIds,
+ mvccCacheId,
+ forUpdateQryOutTx,
+ forUpdateQryTx
+ );
+
+ return new QueryParserResult(
+ newQryDesc,
+ QueryParameters.fromQuery(newQry),
+ remainingQry,
+ paramsMeta,
+ select,
+ null,
+ null
+ );
+ }
+ catch (IgniteCheckedException | SQLException e) {
+ throw new IgniteSQLException("Failed to parse query. " + e.getMessage(), IgniteQueryErrorCode.PARSING, e);
}
finally {
+ qryCtxRegistry.clearThreadLocal();
+
U.close(stmt, log);
}
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java
index b073da7..f9ab6a4 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java
@@ -189,11 +189,6 @@ public class H2PkHashIndex extends GridH2IndexBase {
}
/** {@inheritDoc} */
- @Override public long getRowCountApproximation() {
- return 10_000; // TODO
- }
-
- /** {@inheritDoc} */
@Override public boolean canGetFirstOrLast() {
return false;
}
@@ -203,6 +198,37 @@ public class H2PkHashIndex extends GridH2IndexBase {
throw new UnsupportedOperationException();
}
+ /** {@inheritDoc} */
+ @Override public long totalRowCount(IndexingQueryCacheFilter partsFilter) {
+ CacheDataRowStore.setSkipVersion(true);
+
+ try {
+ Collection<GridCursor<? extends CacheDataRow>> cursors = new ArrayList<>();
+
+ for (IgniteCacheOffheapManager.CacheDataStore store : cctx.offheap().cacheDataStores()) {
+ int part = store.partId();
+
+ if (partsFilter == null || partsFilter.applyPartition(part))
+ cursors.add(store.cursor(cctx.cacheId()));
+ }
+
+ Cursor pkHashCursor = new H2PkHashIndexCursor(cursors.iterator());
+
+ long res = 0;
+
+ while (pkHashCursor.next())
+ res++;
+
+ return res;
+ }
+ catch (IgniteCheckedException e) {
+ throw U.convertException(e);
+ }
+ finally {
+ CacheDataRowStore.setSkipVersion(false);
+ }
+ }
+
/**
* Cursor.
*/
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeClientIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeClientIndex.java
index f578e0d..2f7a29a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeClientIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeClientIndex.java
@@ -18,12 +18,12 @@
package org.apache.ignite.internal.processors.query.h2.database;
import java.util.List;
-
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
-import org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
+import org.apache.ignite.internal.processors.query.h2.opt.H2CacheRow;
+import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter;
import org.h2.engine.Session;
import org.h2.index.Cursor;
import org.h2.index.IndexType;
@@ -83,6 +83,11 @@ public class H2TreeClientIndex extends H2TreeIndexBase {
}
/** {@inheritDoc} */
+ @Override public long totalRowCount(IndexingQueryCacheFilter partsFilter) {
+ throw unsupported();
+ }
+
+ /** {@inheritDoc} */
@Override public int segmentsCount() {
throw unsupported();
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
index 2a327e2..f914703 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
@@ -564,6 +564,24 @@ public class H2TreeIndex extends H2TreeIndexBase {
}
/** {@inheritDoc} */
+ @Override public long totalRowCount(IndexingQueryCacheFilter partsFilter) {
+ try {
+ H2TreeFilterClosure filter = partsFilter == null ? null :
+ new H2TreeFilterClosure(partsFilter, null, cctx, log);
+
+ long cnt = 0;
+
+ for (int seg = 0; seg < segmentsCount(); seg++)
+ cnt += segments[seg].size(filter);
+
+ return cnt;
+ }
+ catch (IgniteCheckedException e) {
+ throw U.convertException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public IndexLookupBatch createLookupBatch(TableFilter[] filters, int filter) {
QueryContext qctx = qryCtxRegistry.getThreadLocal();
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexBase.java
index e7921d3..6a17fec 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndexBase.java
@@ -145,9 +145,4 @@ public abstract class H2TreeIndexBase extends GridH2IndexBase {
@Override public boolean canGetFirstOrLast() {
return true;
}
-
- /** {@inheritDoc} */
- @Override public long getRowCountApproximation() {
- return 10_000; // TODO
- }
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 003aafd..6f10ad7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -20,8 +20,9 @@ package org.apache.ignite.internal.processors.query.h2.opt;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.query.QueryUtils;
-import org.apache.ignite.internal.processors.query.h2.opt.join.CollocationModelMultiplier;
import org.apache.ignite.internal.processors.query.h2.opt.join.CollocationModel;
+import org.apache.ignite.internal.processors.query.h2.opt.join.CollocationModelMultiplier;
+import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter;
import org.h2.engine.Session;
import org.h2.index.BaseIndex;
import org.h2.message.DbException;
@@ -228,4 +229,16 @@ public abstract class GridH2IndexBase extends BaseIndex {
protected QueryContextRegistry queryContextRegistry() {
return tbl.rowDescriptor().indexing().queryContextRegistry();
}
+
+
+ /** {@inheritDoc} */
+ @Override public long getRowCountApproximation() {
+ return tbl.getRowCountApproximation();
+ }
+
+ /**
+ * @param partsFilter Partitions filter.
+ * @return Total row count in the current index for filtered partitions.
+ */
+ public abstract long totalRowCount(IndexingQueryCacheFilter partsFilter);
}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index b59dfb3..250fe2d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -32,6 +32,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.cache.query.QueryRetryException;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheContextInfo;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -41,12 +42,16 @@ import org.apache.ignite.internal.processors.query.QueryField;
import org.apache.ignite.internal.processors.query.QueryUtils;
import org.apache.ignite.internal.processors.query.h2.H2TableDescriptor;
import org.apache.ignite.internal.processors.query.h2.H2Utils;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.IndexRebuildPartialClosure;
import org.apache.ignite.internal.processors.query.h2.database.H2IndexType;
import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex;
import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndexBase;
import org.apache.ignite.internal.processors.query.h2.database.IndexInformation;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter;
+import org.apache.ignite.spi.indexing.IndexingQueryFilter;
import org.h2.command.ddl.CreateTableData;
import org.h2.command.dml.Insert;
import org.h2.engine.DbObject;
@@ -91,6 +96,12 @@ public class GridH2Table extends TableBase {
/** True representation */
private static final int TRUE = 1;
+ /**
+ * Row count statistics update threshold. Stats will be updated when the actual
+ * table size change exceeds this threshold. Should be the number in interval (0,1).
+ */
+ private static final double STATS_UPDATE_THRESHOLD = 0.1; // 10%.
+
/** Cache context info. */
private final GridCacheContextInfo cacheInfo;
@@ -149,6 +160,9 @@ public class GridH2Table extends TableBase {
/** Table version. The version is changed when exclusive lock is acquired (DDL operation is started). */
private final AtomicLong ver = new AtomicLong();
+ /** Table statistics. */
+ private volatile TableStatistics tblStats;
+
/**
* Creates table.
*
@@ -207,6 +221,16 @@ public class GridH2Table extends TableBase {
sysIdxsCnt = idxs.size();
lock = new ReentrantReadWriteLock();
+
+ if (cacheInfo.affinityNode()) {
+ long totalTblSize = rowCount(false);
+
+ size.add(totalTblSize);
+ }
+
+ // Init stats with the dummy values. This prevents us from scanning index with backup filter when
+ // topology may not be initialized yet.
+ tblStats = new TableStatistics(0, 0);
}
/**
@@ -1076,7 +1100,79 @@ public class GridH2Table extends TableBase {
/** {@inheritDoc} */
@Override public long getRowCountApproximation() {
- return size.longValue();
+ if (!localQuery())
+ return 10_000; // Fallback to the previous behaviour.
+
+ refreshStatsIfNeeded();
+
+ return tblStats.primaryRowCount();
+ }
+
+ /**
+ * @return {@code True} if the current query is a local query.
+ */
+ private boolean localQuery() {
+ QueryContext qctx = rowDescriptor().indexing().queryContextRegistry().getThreadLocal();
+
+ assert qctx != null;
+
+ return qctx.local();
+ }
+
+ /**
+ * Refreshes table stats if they are outdated.
+ */
+ private void refreshStatsIfNeeded() {
+ TableStatistics stats = tblStats;
+
+ long statsTotalRowCnt = stats.totalRowCount();
+ long curTotalRowCnt = size.sum();
+
+ // Update stats if total table size changed significantly since the last stats update.
+ if (needRefreshStats(statsTotalRowCnt, curTotalRowCnt)) {
+ long primaryRowCnt = rowCount(true);
+
+ tblStats = new TableStatistics(curTotalRowCnt, primaryRowCnt);
+ }
+ }
+
+ /**
+ * @param statsRowCnt Row count from statistics.
+ * @param actualRowCnt Actual row count.
+ * @return {@code True} if actual table size has changed more than the threshold since last stats update.
+ */
+ private static boolean needRefreshStats(long statsRowCnt, long actualRowCnt) {
+ double delta = U.safeAbs(statsRowCnt - actualRowCnt);
+
+ double relativeChange = delta / (statsRowCnt + 1); // Add 1 to avoid division by zero.
+
+ // Return true if an actual table size has changed more than the threshold since the last stats update.
+ return relativeChange > STATS_UPDATE_THRESHOLD;
+ }
+
+ /**
+ * Retrieves partitions rows count for all segments.
+ *
+ * @param primaryOnly If {@code true}, only primary rows will be counted.
+ * @return Rows count.
+ */
+ private long rowCount(boolean primaryOnly) {
+ IndexingQueryCacheFilter partsFilter = primaryOnly ? backupFilter() : null;
+
+ return ((GridH2IndexBase)getUniqueIndex()).totalRowCount(partsFilter);
+ }
+
+ /**
+ * @return Backup filter for the current topology.
+ */
+ @Nullable private IndexingQueryCacheFilter backupFilter() {
+ IgniteH2Indexing indexing = rowDescriptor().indexing();
+
+ AffinityTopologyVersion topVer = indexing.readyTopologyVersion();
+
+ IndexingQueryFilter filter = indexing.backupFilter(topVer, null);
+
+ return filter.forCache(cacheName());
}
/** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/QueryContext.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/QueryContext.java
index a697bf3..24191ea 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/QueryContext.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/QueryContext.java
@@ -43,6 +43,9 @@ public class QueryContext {
/** */
private final PartitionReservation reservations;
+ /** {@code True} for local queries, {@code false} for distributed ones. */
+ private final boolean loc;
+
/**
* Constructor.
*
@@ -50,6 +53,7 @@ public class QueryContext {
* @param filter Filter.
* @param distributedJoinCtx Distributed join context.
* @param mvccSnapshot MVCC snapshot.
+ * @param loc {@code True} for local queries, {@code false} for distributed ones.
*/
@SuppressWarnings("AssignmentOrReturnOfFieldWithMutableType")
public QueryContext(
@@ -57,13 +61,15 @@ public class QueryContext {
@Nullable IndexingQueryFilter filter,
@Nullable DistributedJoinContext distributedJoinCtx,
@Nullable MvccSnapshot mvccSnapshot,
- @Nullable PartitionReservation reservations
+ @Nullable PartitionReservation reservations,
+ boolean loc
) {
this.segment = segment;
this.filter = filter;
this.distributedJoinCtx = distributedJoinCtx;
this.mvccSnapshot = mvccSnapshot;
this.reservations = reservations;
+ this.loc = loc;
}
/**
@@ -106,6 +112,13 @@ public class QueryContext {
return filter;
}
+ /**
+ * @return {@code True} for local queries, {@code false} for distributed ones.
+ */
+ public boolean local() {
+ return loc;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(QueryContext.class, this);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/TableStatistics.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/TableStatistics.java
new file mode 100644
index 0000000..1d410b9
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/TableStatistics.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.query.h2.opt;
+
+/**
+ * Table statistics class. Used by query optimizer to estimate execution plan cost.
+ */
+public class TableStatistics {
+ /** Total table row count (including primary and backup partitions). */
+ private final long totalRowCnt;
+
+ /** Primary parts row count. */
+ private final long primaryRowCnt;
+
+ /**
+ * @param totalRowCnt Total table row count (including primary and backup partitions).
+ * @param primaryRowCnt Primary parts row count.
+ */
+ public TableStatistics(long totalRowCnt, long primaryRowCnt) {
+ assert totalRowCnt >= 0 && primaryRowCnt >= 0;
+
+ this.totalRowCnt = totalRowCnt;
+ this.primaryRowCnt = primaryRowCnt;
+ }
+
+ /**
+ * @return Total table row count (including primary and backup partitions).
+ */
+ public long totalRowCount() {
+ return totalRowCnt;
+ }
+
+ /**
+ * @return Primary parts row count.
+ */
+ public long primaryRowCount() {
+ return primaryRowCnt;
+ }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index f5c5b43..a747c63 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -345,7 +345,8 @@ public class GridMapQueryExecutor {
h2.backupFilter(topVer, parts),
distributedJoinCtx,
mvccSnapshot,
- reserved
+ reserved,
+ true
);
qryResults = new MapQueryResults(h2, reqId, qrys.size(), mainCctx, lazy, qctx);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index f4e842b..fb4779e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -647,7 +647,8 @@ public class GridReduceQueryExecutor {
null,
null,
null,
- null
+ null,
+ true
);
QueryContextRegistry qryCtxRegistry = h2.queryContextRegistry();
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryBasicTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryBasicTest.java
index 5795b0a..e97390c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryBasicTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryBasicTest.java
@@ -138,24 +138,27 @@ public class CacheMvccSelectForUpdateQueryBasicTest extends CacheMvccAbstractTe
@Override public void beforeTest() throws Exception {
Ignite client = grid(3);
- CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>("dummy")
+ CacheConfiguration<Object, Object> dummyCfg = new CacheConfiguration<>("dummy")
.setSqlSchema("PUBLIC").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
// Create dummy cache as entry point.
- client.getOrCreateCache(ccfg);
+ client.getOrCreateCache(dummyCfg);
- if (segmented) {
- CacheConfiguration<Object, Object> seg = new CacheConfiguration<>("segmented*");
+ String templateName = String.valueOf(cacheMode) + backups + fromClient + segmented;
- seg.setQueryParallelism(4);
+ CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(templateName);
- client.addCacheConfiguration(seg);
- }
+ ccfg.setCacheMode(cacheMode);
+ ccfg.setBackups(backups);
+
+ if (segmented)
+ ccfg.setQueryParallelism(4);
+
+ client.addCacheConfiguration(ccfg);
// Create MVCC table and cache.
runSql(client, "CREATE TABLE person (id INT PRIMARY KEY, name VARCHAR, salary INT) " +
- "WITH \"ATOMICITY=TRANSACTIONAL_SNAPSHOT, BACKUPS=" + backups +
- (segmented ? ", TEMPLATE=segmented" : "") + "\"", false);
+ "WITH \"ATOMICITY=TRANSACTIONAL_SNAPSHOT, TEMPLATE=" + templateName + "\"", false);
runSql(client, "CREATE INDEX salaryIdx ON person(salary)", false);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
index 8f57367..ab38838 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
@@ -34,6 +34,7 @@ import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.processors.cache.index.AbstractIndexingCommonTest;
+import org.junit.Ignore;
import org.junit.Test;
/**
@@ -210,6 +211,7 @@ public class IgniteSqlSegmentedIndexSelfTest extends AbstractIndexingCommonTest
*
* @throws Exception If failed.
*/
+ @Ignore("https://issues.apache.org/jira/browse/IGNITE-11839")
@Test
public void testSegmentedPartitionedWithReplicated() throws Exception {
ignite(0).createCache(cacheConfig(PERSON_CAHE_NAME, true, PersonKey.class, Person.class));
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/RowCountTableStatisticsSurvivesNodeRestartTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/RowCountTableStatisticsSurvivesNodeRestartTest.java
new file mode 100644
index 0000000..2212d0b
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/RowCountTableStatisticsSurvivesNodeRestartTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.query.h2;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.junit.Test;
+
+/**
+ * Checks if table statistics restored after the node has been restarted.
+ */
+public class RowCountTableStatisticsSurvivesNodeRestartTest extends TableStatisticsAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setConsistentId(igniteInstanceName);
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ );
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+
+ startGridsMultiThreaded(1);
+
+ grid(0).getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ runSql("DROP TABLE IF EXISTS big");
+ runSql("DROP TABLE IF EXISTS small");
+
+ runSql("CREATE TABLE big (a INT PRIMARY KEY, b INT, c INT)");
+ runSql("CREATE TABLE small (a INT PRIMARY KEY, b INT, c INT)");
+
+ runSql("CREATE INDEX big_b ON big(b)");
+ runSql("CREATE INDEX small_b ON small(b)");
+
+ runSql("CREATE INDEX big_c ON big(c)");
+ runSql("CREATE INDEX small_c ON small(c)");
+
+ IgniteCache<Integer, Object> cache = grid(0).cache(DEFAULT_CACHE_NAME);
+
+ for (int i = 0; i < BIG_SIZE; i++)
+ runSql("INSERT INTO big(a, b, c) VALUES(" + i + "," + i + "," + i % 10 + ")");
+
+ for (int i = 0; i < SMALL_SIZE; i++)
+ runSql("INSERT INTO small(a, b, c) VALUES(" + i + "," + i + ","+ i % 10 + ")");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void statisticsSurvivesRestart() throws Exception {
+ String sql = "SELECT COUNT(*) FROM t1 JOIN t2 ON t1.c = t2.c " +
+ "WHERE t1.b >= 0 AND t2.b >= 0";
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "big");
+
+ stopGrid(0);
+
+ startGrid(0);
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "big");
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/RowCountTableStatisticsUsageTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/RowCountTableStatisticsUsageTest.java
new file mode 100644
index 0000000..e1cfbcc
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/RowCountTableStatisticsUsageTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+
+/**
+ * Test cases to ensure that proper join order is chosen by H2 optimizer when row count statistics is collected.
+ */
+@RunWith(Parameterized.class)
+public class RowCountTableStatisticsUsageTest extends TableStatisticsAbstractTest {
+ /** */
+ @Parameterized.Parameter(0)
+ public CacheMode cacheMode;
+
+ /**
+ * @return Test parameters.
+ */
+ @Parameterized.Parameters(name = "cacheMode={0}")
+ public static Collection parameters() {
+ return Arrays.asList(new Object[][] {
+ { REPLICATED },
+ { PARTITIONED },
+ });
+ }
+
+ @Override protected void beforeTestsStarted() throws Exception {
+ Ignite node = startGridsMultiThreaded(2);
+
+ node.getOrCreateCache(DEFAULT_CACHE_NAME);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ runSql("DROP TABLE IF EXISTS big");
+ runSql("DROP TABLE IF EXISTS med");
+ runSql("DROP TABLE IF EXISTS small");
+
+ runSql("CREATE TABLE big (a INT PRIMARY KEY, b INT, c INT) WITH \"TEMPLATE=" + cacheMode + "\"");
+ runSql("CREATE TABLE med (a INT PRIMARY KEY, b INT, c INT) WITH \"TEMPLATE=" + cacheMode + "\"");
+ runSql("CREATE TABLE small (a INT PRIMARY KEY, b INT, c INT) WITH \"TEMPLATE=" + cacheMode + "\"");
+
+ runSql("CREATE INDEX big_b ON big(b)");
+ runSql("CREATE INDEX med_b ON med(b)");
+ runSql("CREATE INDEX small_b ON small(b)");
+
+ runSql("CREATE INDEX big_c ON big(c)");
+ runSql("CREATE INDEX med_c ON med(c)");
+ runSql("CREATE INDEX small_c ON small(c)");
+
+ for (int i = 0; i < BIG_SIZE; i++)
+ runSql("INSERT INTO big(a, b, c) VALUES(" + i + "," + i + "," + i % 10 + ")");
+
+ for (int i = 0; i < MED_SIZE; i++)
+ runSql("INSERT INTO med(a, b, c) VALUES(" + i + "," + i + "," + i % 10 + ")");
+
+ for (int i = 0; i < SMALL_SIZE; i++)
+ runSql("INSERT INTO small(a, b, c) VALUES(" + i + "," + i + ","+ i % 10 + ")");
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void compareJoinsWithConditionsOnBothTables() {
+ String sql = "SELECT COUNT(*) FROM t1 JOIN t2 ON t1.c = t2.c " +
+ "WHERE t1.b >= 0 AND t2.b >= 0";
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "big");
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void compareJoinsWithoutConditions() {
+ String sql = "SELECT COUNT(*) FROM t1 JOIN t2 ON t1.c = t2.c";
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "big", "small");
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void compareJoinsConditionSingleTable() {
+ final String sql = "SELECT * FROM t1 JOIN t2 ON t1.c = t2.c WHERE t1.b >= 0";
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "big", "small");
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void compareJoinsThreeTablesNoConditions() {
+ String sql = "SELECT * FROM t1 JOIN t2 ON t1.c = t2.c JOIN t3 ON t3.c = t2.c ";
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "big", "med", "small");
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "big", "med");
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "med", "big");
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "med", "big", "small");
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void compareJoinsThreeTablesConditionsOnAllTables() {
+ String sql = "SELECT * FROM t1 JOIN t2 ON t1.c = t2.c JOIN t3 ON t3.c = t2.c " +
+ " WHERE t1.b >= 0 AND t2.b >= 0 AND t3.b >= 0";
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "big", "med", "small");
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "big", "med");
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "med", "big");
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "med", "big", "small");
+ }
+
+ /**
+ * Checks if statistics is updated when table size is changed.
+ */
+ @Test
+ public void checkUpdateStatisticsOnTableSizeChange() {
+ // t2 size is bigger than t1
+ String sql = "SELECT COUNT(*) FROM t2 JOIN t1 ON t1.c = t2.c " +
+ "WHERE t1.b > " + 0 + " AND t2.b > " + 0;
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "big");
+
+ // Make small table bigger than a big table
+ for (int i = SMALL_SIZE; i < BIG_SIZE * 2; i++)
+ runSql("INSERT INTO small(a, b, c) VALUES(" + i + "," + i + "," + i % 10 + ")");
+
+ // t1 size is now bigger than t2
+ sql = "SELECT COUNT(*) FROM t1 JOIN t2 ON t1.c = t2.c " +
+ "WHERE t1.b > " + 0 + " AND t2.b > " + 0;
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "small", "big");
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testStatisticsAfterRebalance() throws Exception {
+ String sql = "SELECT COUNT(*) FROM t1 JOIN t2 ON t1.c = t2.c";
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "big", "small");
+
+ startGrid(3);
+
+ try {
+ awaitPartitionMapExchange();
+
+ grid(3).context()
+ .cache()
+ .context()
+ .cacheContext(CU.cacheId("SQL_PUBLIC_BIG"))
+ .preloader()
+ .rebalanceFuture()
+ .get(10_000);
+
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(1), sql, "big", "small");
+ checkOptimalPlanChosenForDifferentJoinOrders(grid(0), sql, "big", "small");
+ }
+ finally {
+ stopGrid(3);
+ }
+ }
+}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/TableStatisticsAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/TableStatisticsAbstractTest.java
new file mode 100644
index 0000000..acc48c1
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/TableStatisticsAbstractTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Base test for table statistics.
+ */
+public abstract class TableStatisticsAbstractTest extends GridCommonAbstractTest {
+ /** */
+ static final int BIG_SIZE = 1000;
+
+ /** */
+ static final int MED_SIZE = 500;
+
+ /** */
+ static final int SMALL_SIZE = 100;
+
+ static {
+ assertTrue(SMALL_SIZE < MED_SIZE && MED_SIZE < BIG_SIZE);
+ }
+
+ /**
+ * Compares different orders of joins for the given query.
+ *
+ * @param sql Query.
+ * @param tbls Table names.
+ */
+ protected void checkOptimalPlanChosenForDifferentJoinOrders(Ignite grid, String sql, String... tbls) {
+ String directOrder = replaceTablePlaceholders(sql, tbls);
+
+ if (log.isDebugEnabled())
+ log.debug("Direct join order=" + directOrder);
+
+ ensureOptimalPlanChosen(grid, directOrder);
+
+ // Reverse tables order.
+ List<String> dirOrdTbls = Arrays.asList(tbls);
+
+ Collections.reverse(dirOrdTbls);
+
+ String reversedOrder = replaceTablePlaceholders(sql, (String[]) dirOrdTbls.toArray());
+
+ if (log.isDebugEnabled())
+ log.debug("Reversed join order=" + reversedOrder);
+
+ ensureOptimalPlanChosen(grid, reversedOrder);
+ }
+
+ /**
+ * Compares join orders by actually scanned rows. Join command is run twice:
+ * with {@code enforceJoinOrder = true} and without. The latest allows join order optimization
+ * based or table row count.
+ *
+ * Actual scan row count is obtained from the EXPLAIN ANALYZE command result.
+ */
+ private void ensureOptimalPlanChosen(Ignite grid, String sql, String ... tbls) {
+ int cntNoStats = runLocalExplainAnalyze(grid, true, sql);
+
+ int cntStats = runLocalExplainAnalyze(grid, false, sql);
+
+ String res = "Scanned rows count [noStats=" + cntNoStats + ", withStats=" + cntStats +
+ ", diff=" + (cntNoStats - cntStats) + ']';
+
+ if (log.isInfoEnabled())
+ log.info(res);
+
+ assertTrue(res, cntStats <= cntNoStats);
+ }
+
+ /**
+ * Runs local join sql in EXPLAIN ANALYZE mode and extracts actual scanned row count from the result.
+ *
+ * @param enfJoinOrder Enforce join order flag.
+ * @param sql Sql string.
+ * @return Actual scanned rows count.
+ */
+ private int runLocalExplainAnalyze(Ignite grid, boolean enfJoinOrder, String sql) {
+ List<List<?>> res = grid.cache(DEFAULT_CACHE_NAME)
+ .query(new SqlFieldsQuery("EXPLAIN ANALYZE " + sql)
+ .setEnforceJoinOrder(enfJoinOrder)
+ .setLocal(true))
+ .getAll();
+
+ if (log.isDebugEnabled())
+ log.debug("ExplainAnalyze enfJoinOrder=" + enfJoinOrder + ", res=" + res);
+
+ return extractScanCountFromExplain(res);
+ }
+
+ /**
+ * Extracts actual scanned rows count from EXPLAIN ANALYZE result.
+ *
+ * @param res EXPLAIN ANALYZE result.
+ * @return actual scanned rows count.
+ */
+ private int extractScanCountFromExplain(List<List<?>> res) {
+ String explainRes = (String)res.get(0).get(0);
+
+ // Extract scan count from EXPLAIN ANALYZE with regex: return all numbers after "scanCount: ".
+ Matcher m = Pattern.compile("scanCount: (?=(\\d+))").matcher(explainRes);
+
+ int scanCnt = 0;
+
+ while(m.find())
+ scanCnt += Integer.valueOf(m.group(1));
+
+ return scanCnt;
+ }
+
+ /**
+ * @param sql Statement.
+ */
+ protected void runSql(String sql) {
+ grid(0).cache(DEFAULT_CACHE_NAME).query(new SqlFieldsQuery(sql)).getAll();
+ }
+
+ /**
+ * Replaces table placeholders like "t1", "t2" and others with actual table names in the SQL query.
+ *
+ * @param sql Sql query.
+ * @param tbls Actual table names.
+ * @return Sql with place holders replaced by the actual names.
+ */
+ private static String replaceTablePlaceholders(String sql, String ... tbls) {
+ assert !sql.contains("t0");
+
+ int i = 0;
+
+ for (String tbl : tbls) {
+ String tblPlaceHolder = "t" + (++i);
+
+ assert sql.contains(tblPlaceHolder);
+
+ sql = sql.replace(tblPlaceHolder, tbl);
+ }
+
+ assert !sql.contains("t" + (i + 1));
+
+ return sql;
+ }
+
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index 47b636f..e88d04a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -47,6 +47,8 @@ import org.apache.ignite.internal.processors.query.GridQueryProcessor;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.QueryUtils;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.opt.QueryContext;
+import org.apache.ignite.internal.processors.query.h2.opt.QueryContextRegistry;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.GridTestUtils;
@@ -1041,7 +1043,46 @@ public class GridQueryParsingTest extends AbstractIndexingCommonTest {
private <T extends Prepared> T parse(String sql) throws Exception {
Session ses = (Session)connection().getSession();
- return (T)ses.prepare(sql);
+ setQueryContext();
+
+ try {
+ return (T)ses.prepare(sql);
+ }
+ finally {
+ clearQueryContext();
+ }
+ }
+
+ /**
+ * Sets thread local query context.
+ */
+ private void setQueryContext() {
+ QueryContextRegistry qryCtxRegistry = indexing().queryContextRegistry();
+
+ QueryContext qctx = new QueryContext(
+ 0,
+ null,
+ null,
+ null,
+ null,
+ true
+ );
+
+ qryCtxRegistry.setThreadLocal(qctx);
+ }
+
+ /**
+ * Clears thread local query context.
+ */
+ private void clearQueryContext() {
+ indexing().queryContextRegistry().clearThreadLocal();
+ }
+
+ /**
+ * @return H2 indexing manager.
+ */
+ private IgniteH2Indexing indexing() {
+ return (IgniteH2Indexing)((IgniteEx)ignite).context().query().getIndexing();
}
/**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
index 6a78621..9f82f14 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
@@ -216,6 +216,8 @@ import org.apache.ignite.internal.processors.query.h2.H2ResultSetIteratorNullify
import org.apache.ignite.internal.processors.query.h2.IgniteSqlBigIntegerKeyTest;
import org.apache.ignite.internal.processors.query.h2.IgniteSqlQueryMinMaxTest;
import org.apache.ignite.internal.processors.query.h2.QueryDataPageScanTest;
+import org.apache.ignite.internal.processors.query.h2.RowCountTableStatisticsSurvivesNodeRestartTest;
+import org.apache.ignite.internal.processors.query.h2.RowCountTableStatisticsUsageTest;
import org.apache.ignite.internal.processors.query.h2.ThreadLocalObjectPoolSelfTest;
import org.apache.ignite.internal.processors.query.h2.sql.BaseH2CompareQueryTest;
import org.apache.ignite.internal.processors.query.h2.sql.ExplainSelfTest;
@@ -570,6 +572,10 @@ import org.junit.runners.Suite;
KillQueryFromClientTest.class,
KillQueryOnClientDisconnectTest.class,
+ // Table statistics.
+ RowCountTableStatisticsUsageTest.class,
+ RowCountTableStatisticsSurvivesNodeRestartTest.class
+
})
public class IgniteBinaryCacheQueryTestSuite {
}