You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/11/24 11:30:58 UTC
[2/8] ignite git commit: ignite-sql-cache-stmt - wip
ignite-sql-cache-stmt - wip
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3becf611
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3becf611
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3becf611
Branch: refs/heads/ignite-sql-opt-2
Commit: 3becf611b5695c827f8ff5ed269590957cfd0002
Parents: 53f0c1d
Author: S.Vladykin <sv...@gridgain.com>
Authored: Tue Nov 24 01:01:32 2015 +0300
Committer: S.Vladykin <sv...@gridgain.com>
Committed: Tue Nov 24 01:01:32 2015 +0300
----------------------------------------------------------------------
.../cache/query/GridCacheSqlQuery.java | 42 ++++++++++
.../cache/query/GridCacheTwoStepQuery.java | 20 ++++-
.../processors/query/h2/IgniteH2Indexing.java | 82 ++++++++++++++------
.../query/h2/sql/GridSqlQuerySplitter.java | 73 +++++++++++------
.../h2/twostep/GridReduceQueryExecutor.java | 3 +-
5 files changed, 172 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3becf611/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
index be7bbe9..c955eb8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
@@ -55,6 +55,16 @@ public class GridCacheSqlQuery implements Message {
/** */
@GridToStringInclude
@GridDirectTransient
+ private int[] paramIdxs;
+
+ /** */
+ @GridToStringInclude
+ @GridDirectTransient
+ private int paramsSize;
+
+ /** */
+ @GridToStringInclude
+ @GridDirectTransient
private LinkedHashMap<String, ?> cols;
/** Field kept for backward compatibility. */
@@ -77,6 +87,14 @@ public class GridCacheSqlQuery implements Message {
this.qry = qry;
this.params = F.isEmpty(params) ? EMPTY_PARAMS : params;
+ paramsSize = this.params.length;
+ }
+
+ /**
+ * @param paramIdxs Parameter indexes.
+ */
+ public void parameterIndexes(int[] paramIdxs) {
+ this.paramIdxs = paramIdxs;
}
/**
@@ -222,4 +240,28 @@ public class GridCacheSqlQuery implements Message {
@Override public byte fieldsCount() {
return 3;
}
+
+ /**
+ * @param args Arguments.
+ * @return Copy.
+ */
+ public GridCacheSqlQuery copy(Object[] args) {
+ GridCacheSqlQuery copy = new GridCacheSqlQuery();
+
+ copy.qry = qry;
+ copy.cols = cols;
+ copy.paramIdxs = paramIdxs;
+ copy.paramsSize = paramsSize;
+
+ if (F.isEmpty(args))
+ copy.params = EMPTY_PARAMS;
+ else {
+ copy.params = new Object[paramsSize];
+
+ for (int paramIdx : paramIdxs)
+ copy.params[paramIdx] = args[paramIdx];
+ }
+
+ return copy;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/3becf611/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
index 4b5fe22..a977f13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
@@ -89,9 +89,12 @@ public class GridCacheTwoStepQuery {
/**
* @param qry SQL Query.
+ * @return {@code this}.
*/
- public void addMapQuery(GridCacheSqlQuery qry) {
+ public GridCacheTwoStepQuery addMapQuery(GridCacheSqlQuery qry) {
mapQrys.add(qry);
+
+ return this;
}
/**
@@ -122,6 +125,21 @@ public class GridCacheTwoStepQuery {
this.spaces = spaces;
}
+ /**
+ * @param args New arguments to copy with.
+ * @return Copy.
+ */
+ public GridCacheTwoStepQuery copy(Object[] args) {
+ assert !explain;
+
+ GridCacheTwoStepQuery copy = new GridCacheTwoStepQuery(spaces, rdc.copy(args));
+ copy.pageSize = pageSize;
+ for (int i = 0; i < mapQrys.size(); i++)
+ copy.mapQrys.add(mapQrys.get(i).copy(args));
+
+ return copy;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridCacheTwoStepQuery.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3becf611/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 6cf65b6..7e1c415 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -99,6 +99,7 @@ import org.apache.ignite.internal.util.lang.GridCloseableIterator;
import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeGuard;
import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -276,6 +277,11 @@ public class IgniteH2Indexing implements GridQueryIndexing {
};
/**
+ * TODO add some kind of eviction like LRU
+ */
+ private final ConcurrentMap<T2<String, String>, TwoStepCachedQuery> twoStepCache = new ConcurrentHashMap8<>();
+
+ /**
* @param space Space.
* @return Connection.
*/
@@ -301,11 +307,15 @@ public class IgniteH2Indexing implements GridQueryIndexing {
PreparedStatement stmt = cache.get(sql);
- if (stmt != null)
+ if (stmt != null && !stmt.isClosed()) {
+ assert stmt.getConnection() == c;
+
return stmt;
+ }
stmt = c.prepareStatement(sql);
+ // TODO close the statement on evict from cache
cache.put(sql, stmt);
return stmt;
@@ -965,38 +975,47 @@ public class IgniteH2Indexing implements GridQueryIndexing {
/** {@inheritDoc} */
@Override public QueryCursor<List<?>> queryTwoStep(GridCacheContext<?,?> cctx, SqlFieldsQuery qry) {
- String space = cctx.name();
- String sqlQry = qry.getSql();
+ final String space = cctx.name();
+ final String sqlQry = qry.getSql();
Connection c = connectionForSpace(space);
- PreparedStatement stmt;
+ GridCacheTwoStepQuery twoStepQry;
+ List<GridQueryFieldMetadata> meta;
- try {
- stmt = prepareStatement(c, sqlQry, true);
- }
- catch (SQLException e) {
- throw new CacheException("Failed to parse query: " + sqlQry, e);
- }
+ final T2<String, String> cachedQryKey = new T2<>(space, sqlQry);
+ TwoStepCachedQuery cachedQry = twoStepCache.get(cachedQryKey);
- try {
- bindParameters(stmt, F.asList(qry.getArgs()));
- }
- catch (IgniteCheckedException e) {
- throw new CacheException("Failed to bind parameters: [qry=" + sqlQry + ", params=" +
- Arrays.deepToString(qry.getArgs()) + "]", e);
+ if (cachedQry != null) {
+ twoStepQry = cachedQry.twoStepQuery.copy(qry.getArgs());
+ meta = cachedQry.meta;
}
+ else {
+ PreparedStatement stmt;
- GridCacheTwoStepQuery twoStepQry;
- List<GridQueryFieldMetadata> meta;
+ try {
+ stmt = prepareStatement(c, sqlQry, true);
+ }
+ catch (SQLException e) {
+ throw new CacheException("Failed to parse query: " + sqlQry, e);
+ }
- try {
- twoStepQry = GridSqlQuerySplitter.split((JdbcPreparedStatement)stmt, qry.getArgs(), qry.isCollocated());
+ try {
+ bindParameters(stmt, F.asList(qry.getArgs()));
+ }
+ catch (IgniteCheckedException e) {
+ throw new CacheException("Failed to bind parameters: [qry=" + sqlQry + ", params=" +
+ Arrays.deepToString(qry.getArgs()) + "]", e);
+ }
- meta = meta(stmt.getMetaData());
- }
- catch (SQLException e) {
- throw new CacheException(e);
+ try {
+ twoStepQry = GridSqlQuerySplitter.split((JdbcPreparedStatement)stmt, qry.getArgs(), qry.isCollocated());
+
+ meta = meta(stmt.getMetaData());
+ }
+ catch (SQLException e) {
+ throw new CacheException(e);
+ }
}
if (log.isDebugEnabled())
@@ -1008,10 +1027,25 @@ public class IgniteH2Indexing implements GridQueryIndexing {
cursor.fieldsMeta(meta);
+ if (cachedQry == null && !twoStepQry.explain()) {
+ cachedQry = new TwoStepCachedQuery();
+ cachedQry.meta = meta;
+ cachedQry.twoStepQuery = twoStepQry.copy(null);
+ twoStepCache.putIfAbsent(cachedQryKey, cachedQry);
+ }
+
return cursor;
}
/**
+ * Cached two-step query.
+ */
+ private static final class TwoStepCachedQuery {
+ List<GridQueryFieldMetadata> meta;
+ GridCacheTwoStepQuery twoStepQuery;
+ }
+
+ /**
* Sets filters for current thread. Must be set to not null value
* before executeQuery and reset to null after in finally block since it signals
* to table that it should return content without expired values.
http://git-wip-us.apache.org/repos/asf/ignite/blob/3becf611/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 7b89824..ac47b50 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -28,6 +28,7 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.util.typedef.F;
import org.h2.jdbc.JdbcPreparedStatement;
+import org.h2.util.IntArray;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.AVG;
@@ -224,13 +225,23 @@ public class GridSqlQuerySplitter {
rdcQry.distinct(true);
}
- // Build resulting two step query.
- GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(spaces, new GridCacheSqlQuery(rdcQry.getSQL(),
- findParams(rdcQry, params, new ArrayList<>()).toArray()));
+ IntArray paramIdxs = new IntArray(params.length);
+
+ GridCacheSqlQuery rdc = new GridCacheSqlQuery(rdcQry.getSQL(),
+ findParams(rdcQry, params, new ArrayList<>(), paramIdxs).toArray());
+
+ rdc.parameterIndexes(toIntArray(paramIdxs));
+
+ paramIdxs = new IntArray(params.length);
- res.addMapQuery(new GridCacheSqlQuery(mapQry.getSQL(),
- findParams(mapQry, params, new ArrayList<>(params.length)).toArray())
- .columns(collectColumns(mapExps)));
+ GridCacheSqlQuery map = new GridCacheSqlQuery(mapQry.getSQL(),
+ findParams(mapQry, params, new ArrayList<>(params.length), paramIdxs).toArray())
+ .columns(collectColumns(mapExps));
+
+ map.parameterIndexes(toIntArray(paramIdxs));
+
+ // Build resulting two step query.
+ GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(spaces, rdc).addMapQuery(map);
res.explain(explain);
@@ -238,6 +249,16 @@ public class GridSqlQuerySplitter {
}
/**
+ * @param arr Integer array.
+ * @return Primitive int array.
+ */
+ private static int[] toIntArray(IntArray arr) {
+ int[] res = new int[arr.size()];
+ arr.toArray(res);
+ return res;
+ }
+
+ /**
* @param cols Columns from SELECT clause.
* @return Map of columns with types.
*/
@@ -341,19 +362,21 @@ public class GridSqlQuerySplitter {
* @param qry Select.
* @param params Parameters.
* @param target Extracted parameters.
+ * @param paramIdxs Parameter indexes.
* @return Extracted parameters list.
*/
- private static List<Object> findParams(GridSqlQuery qry, Object[] params, ArrayList<Object> target) {
+ private static List<Object> findParams(GridSqlQuery qry, Object[] params, ArrayList<Object> target,
+ IntArray paramIdxs) {
if (qry instanceof GridSqlSelect)
- return findParams((GridSqlSelect)qry, params, target);
+ return findParams((GridSqlSelect)qry, params, target, paramIdxs);
GridSqlUnion union = (GridSqlUnion)qry;
- findParams(union.left(), params, target);
- findParams(union.right(), params, target);
+ findParams(union.left(), params, target, paramIdxs);
+ findParams(union.right(), params, target, paramIdxs);
- findParams(qry.limit(), params, target);
- findParams(qry.offset(), params, target);
+ findParams(qry.limit(), params, target, paramIdxs);
+ findParams(qry.offset(), params, target, paramIdxs);
return target;
}
@@ -362,22 +385,24 @@ public class GridSqlQuerySplitter {
* @param qry Select.
* @param params Parameters.
* @param target Extracted parameters.
+ * @param paramIdxs Parameter indexes.
* @return Extracted parameters list.
*/
- private static List<Object> findParams(GridSqlSelect qry, Object[] params, ArrayList<Object> target) {
+ private static List<Object> findParams(GridSqlSelect qry, Object[] params, ArrayList<Object> target,
+ IntArray paramIdxs) {
if (params.length == 0)
return target;
for (GridSqlElement el : qry.columns(false))
- findParams(el, params, target);
+ findParams(el, params, target, paramIdxs);
- findParams(qry.from(), params, target);
- findParams(qry.where(), params, target);
+ findParams(qry.from(), params, target, paramIdxs);
+ findParams(qry.where(), params, target, paramIdxs);
// Don't search in GROUP BY and HAVING since they expected to be in select list.
- findParams(qry.limit(), params, target);
- findParams(qry.offset(), params, target);
+ findParams(qry.limit(), params, target, paramIdxs);
+ findParams(qry.offset(), params, target, paramIdxs);
return target;
}
@@ -386,15 +411,17 @@ public class GridSqlQuerySplitter {
* @param el Element.
* @param params Parameters.
* @param target Extracted parameters.
+ * @param paramIdxs Parameter indexes.
*/
- private static void findParams(@Nullable GridSqlElement el, Object[] params, ArrayList<Object> target) {
+ private static void findParams(@Nullable GridSqlElement el, Object[] params, ArrayList<Object> target,
+ IntArray paramIdxs) {
if (el == null)
return;
if (el instanceof GridSqlParameter) {
// H2 Supports queries like "select ?5" but first 4 non-existing parameters are need to be set to any value.
// Here we will set them to NULL.
- int idx = ((GridSqlParameter)el).index();
+ final int idx = ((GridSqlParameter)el).index();
while (target.size() < idx)
target.add(null);
@@ -409,12 +436,14 @@ public class GridSqlQuerySplitter {
target.add(param);
else
target.set(idx, param);
+
+ paramIdxs.add(idx);
}
else if (el instanceof GridSqlSubquery)
- findParams(((GridSqlSubquery)el).select(), params, target);
+ findParams(((GridSqlSubquery)el).select(), params, target, paramIdxs);
else
for (GridSqlElement child : el)
- findParams(child, params, target);
+ findParams(child, params, target, paramIdxs);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/3becf611/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index c8e0d78..000d249 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -594,7 +594,8 @@ public class GridReduceQueryExecutor {
GridCacheSqlQuery rdc = qry.reduceQuery();
- res = h2.executeSqlQueryWithTimer(space, r.conn, rdc.query(), F.asList(rdc.parameters()), true);
+ // Statement caching is prohibited here because we can't guarantee correct merge index reuse.
+ res = h2.executeSqlQueryWithTimer(space, r.conn, rdc.query(), F.asList(rdc.parameters()), false);
}
for (GridMergeTable tbl : r.tbls) {