You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2017/03/01 14:32:44 UTC
[01/50] [abbrv] ignite git commit: Fixed NPE in
IgniteProcessProxy.kill() (cherry picked from commit 11bc058)
Repository: ignite
Updated Branches:
refs/heads/master 5456caffc -> cc6257f8c
Fixed NPE in IgniteProcessProxy.kill()
(cherry picked from commit 11bc058)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9eb26bd8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9eb26bd8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9eb26bd8
Branch: refs/heads/master
Commit: 9eb26bd83ff57980adb929dddf7d4bd62810bf51
Parents: 7a65281
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Fri Feb 17 18:45:17 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Feb 17 19:02:07 2017 +0700
----------------------------------------------------------------------
.../testframework/junits/multijvm/IgniteProcessProxy.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9eb26bd8/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
index 9bb5205..7c62208 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
@@ -234,9 +234,12 @@ public class IgniteProcessProxy implements IgniteEx {
* @param gridName Grid name.
*/
public static void kill(String gridName) {
+ A.notNull(gridName, "gridName");
+
IgniteProcessProxy proxy = gridProxies.get(gridName);
- A.notNull(gridName, "gridName");
+ if (proxy == null)
+ return;
try {
proxy.getProcess().kill();
[24/50] [abbrv] ignite git commit: Minor. .
Posted by av...@apache.org.
Minor. .
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f4d36cb6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f4d36cb6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f4d36cb6
Branch: refs/heads/master
Commit: f4d36cb683b82bc0d059f183a586be4662818bbc
Parents: 2e90a07
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 15:29:52 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 15:29:52 2017 +0300
----------------------------------------------------------------------
.../internal/processors/query/h2/sql/GridSqlQuerySplitter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f4d36cb6/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 165b6b8..779e565 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -1628,7 +1628,7 @@ public class GridSqlQuerySplitter {
findParams(((GridSqlSubquery)el).subquery(), params, target, paramIdxs);
else {
for (int i = 0; i < el.size(); i++)
- findParams(el.child(i), params, target, paramIdxs);
+ findParams((GridSqlAst)el.child(i), params, target, paramIdxs);
}
}
[13/50] [abbrv] ignite git commit: ignite-3860 - fixes
Posted by av...@apache.org.
ignite-3860 - fixes
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/166e65c0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/166e65c0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/166e65c0
Branch: refs/heads/master
Commit: 166e65c0f946529852d72bf8a74f12c4130a64dd
Parents: 43bf89f
Author: Sergi Vladykin <se...@gmail.com>
Authored: Tue Feb 21 11:08:27 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Tue Feb 21 11:08:27 2017 +0300
----------------------------------------------------------------------
.../query/h2/sql/GridSqlQuerySplitter.java | 3 ++-
.../h2/twostep/GridReduceQueryExecutor.java | 26 ++++++++++----------
2 files changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/166e65c0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 5352c87..165b6b8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -209,7 +209,8 @@ public class GridSqlQuerySplitter {
boolean allCollocated = true;
for (GridCacheSqlQuery mapSqlQry : splitter.mapSqlQrys) {
- Prepared prepared = optimize(h2, conn, mapSqlQry.query(), params, true, enforceJoinOrder);
+ Prepared prepared = optimize(h2, conn, mapSqlQry.query(), mapSqlQry.parameters(),
+ true, enforceJoinOrder);
allCollocated &= isCollocated((Query)prepared);
http://git-wip-us.apache.org/repos/asf/ignite/blob/166e65c0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 128ca8e..f813cae 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -99,6 +99,7 @@ import org.jsr166.ConcurrentHashMap8;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
+import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.setupConnection;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter.mergeTableIdentifier;
@@ -674,23 +675,22 @@ public class GridReduceQueryExecutor {
if (skipMergeTbl) {
List<List<?>> res = new ArrayList<>();
- assert r.idxs.size() == 1 : r.idxs;
+ // Simple UNION ALL can have multiple indexes.
+ for (GridMergeIndex idx : r.idxs) {
+ Cursor cur = idx.findInStream(null, null);
- GridMergeIndex idx = r.idxs.get(0);
+ while (cur.next()) {
+ Row row = cur.get();
- Cursor cur = idx.findInStream(null, null);
+ int cols = row.getColumnCount();
- while (cur.next()) {
- Row row = cur.get();
+ List<Object> resRow = new ArrayList<>(cols);
- int cols = row.getColumnCount();
+ for (int c = 0; c < cols; c++)
+ resRow.add(row.getValue(c).getObject());
- List<Object> resRow = new ArrayList<>(cols);
-
- for (int c = 0; c < cols; c++)
- resRow.add(row.getValue(c).getObject());
-
- res.add(resRow);
+ res.add(resRow);
+ }
}
resIter = res.iterator();
@@ -700,7 +700,7 @@ public class GridReduceQueryExecutor {
UUID locNodeId = ctx.localNodeId();
- h2.setupConnection(r.conn, false, enforceJoinOrder);
+ setupConnection(r.conn, false, enforceJoinOrder);
GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, qryReqId, REDUCE)
.pageSize(r.pageSize).distributedJoins(false));
[10/50] [abbrv] ignite git commit: ignite-3860 - merge to 1.9 #
Conflicts:
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index e164315..5352c87 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -17,34 +17,53 @@
package org.apache.ignite.internal.processors.query.h2.sql;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
-import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
-import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.typedef.F;
import org.h2.command.Prepared;
+import org.h2.command.dml.Query;
+import org.h2.command.dml.SelectUnion;
import org.h2.jdbc.JdbcPreparedStatement;
-import org.h2.table.Column;
-import org.h2.table.IndexColumn;
import org.h2.util.IntArray;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.setupConnection;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2CollocationModel.isCollocated;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst.TRUE;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.AVG;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.CAST;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.COUNT;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.MAX;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.MIN;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType.SUM;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlJoin.LEFT_TABLE_CHILD;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlJoin.ON_CHILD;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlJoin.RIGHT_TABLE_CHILD;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlPlaceholder.EMPTY;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery.LIMIT_CHILD;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery.OFFSET_CHILD;
import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser.prepared;
-import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser.query;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect.FROM_CHILD;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect.WHERE_CHILD;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect.childIndexForColumn;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlUnion.LEFT_CHILD;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlUnion.RIGHT_CHILD;
import static org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.toArray;
/**
@@ -52,10 +71,10 @@ import static org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQ
*/
public class GridSqlQuerySplitter {
/** */
- private static final String TABLE_SCHEMA = "PUBLIC";
+ private static final String MERGE_TABLE_SCHEMA = "PUBLIC"; // Schema PUBLIC must always exist.
/** */
- private static final String TABLE_PREFIX = "__T";
+ private static final String MERGE_TABLE_PREFIX = "__T";
/** */
private static final String COLUMN_PREFIX = "__C";
@@ -63,264 +82,1129 @@ public class GridSqlQuerySplitter {
/** */
private static final String HAVING_COLUMN = "__H";
+ /** */
+ private static final String UNIQUE_TABLE_ALIAS_SUFFIX = "__Z";
+
+ /** */
+ private int nextTblAliasId;
+
+ /** */
+ private int splitId = -1; // The first one will be 0.
+
+ /** */
+ private Set<String> schemas = new HashSet<>();
+
+ /** */
+ private Set<String> tbls = new HashSet<>();
+
+ /** */
+ private boolean rdcQrySimple;
+
+ /** */
+ private GridCacheSqlQuery rdcSqlQry;
+
+ /** */
+ private List<GridCacheSqlQuery> mapSqlQrys = new ArrayList<>();
+
+ /** */
+ private Object[] params;
+
+ /** */
+ private boolean collocatedGrpBy;
+
+ /** */
+ private IdentityHashMap<GridSqlAst, GridSqlAlias> uniqueFromAliases = new IdentityHashMap<>();
+
+ /**
+ * @param params Query parameters.
+ * @param collocatedGrpBy If it is a collocated GROUP BY query.
+ */
+ public GridSqlQuerySplitter(Object[] params, boolean collocatedGrpBy) {
+ this.params = params;
+ this.collocatedGrpBy = collocatedGrpBy;
+ }
+
/**
* @param idx Index of table.
- * @return Table.
+ * @return Merge table.
*/
- public static GridSqlTable table(int idx) {
- return new GridSqlTable(TABLE_SCHEMA, TABLE_PREFIX + idx);
+ private static GridSqlTable mergeTable(int idx) {
+ return new GridSqlTable(MERGE_TABLE_SCHEMA, MERGE_TABLE_PREFIX + idx);
+ }
+
+ /**
+ * @param idx Table index.
+ * @return Merge table name.
+ */
+ public static String mergeTableIdentifier(int idx) {
+ return mergeTable(idx).getSQL();
}
/**
* @param idx Index of column.
* @return Generated by index column alias.
*/
- private static String columnName(int idx) {
- return COLUMN_PREFIX + idx;
+ private String columnName(int idx) {
+ // We must have unique columns for each split to avoid name clashes.
+ return COLUMN_PREFIX + splitId + '_' + idx;
+ }
+
+ /**
+ * @param stmt Prepared statement.
+ * @param params Parameters.
+ * @param collocatedGrpBy Whether the query has collocated GROUP BY keys.
+ * @param distributedJoins If distributed joins enabled.
+ * @param enforceJoinOrder Enforce join order.
+ * @param h2 Indexing.
+ * @return Two step query.
+ * @throws SQLException If failed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static GridCacheTwoStepQuery split(
+ JdbcPreparedStatement stmt,
+ Object[] params,
+ boolean collocatedGrpBy,
+ boolean distributedJoins,
+ boolean enforceJoinOrder,
+ IgniteH2Indexing h2
+ ) throws SQLException, IgniteCheckedException {
+ if (params == null)
+ params = GridCacheSqlQuery.EMPTY_PARAMS;
+
+ // Here we will just do initial query parsing. Do not use optimized
+ // subqueries because we do not have unique FROM aliases yet.
+ GridSqlQuery qry = parse(prepared(stmt), false);
+
+ String originalSql = qry.getSQL();
+
+ final boolean explain = qry.explain();
+
+ qry.explain(false);
+
+ GridSqlQuerySplitter splitter = new GridSqlQuerySplitter(params, collocatedGrpBy);
+
+ // Normalization will generate unique aliases for all the table filters in FROM.
+ // Also it will collect all tables and schemas from the query.
+ splitter.normalizeQuery(qry);
+
+ Connection conn = stmt.getConnection();
+
+ // Here we will have correct normalized AST with optimized join order.
+ // The distributedJoins parameter is ignored because it is not relevant for
+ // the REDUCE query optimization.
+ qry = parse(optimize(h2, conn, qry.getSQL(), params, false, enforceJoinOrder),
+ true);
+
+ // Do the actual query split. We will update the original query AST, need to be careful.
+ splitter.splitQuery(qry);
+
+ assert !F.isEmpty(splitter.mapSqlQrys): "map"; // We must have at least one map query.
+ assert splitter.rdcSqlQry != null: "rdc"; // We must have a reduce query.
+
+ // If we have distributed joins, then we have to optimize all MAP side queries
+ // to have a correct join order with respect to batched joins and check if we need
+ // distributed joins at all.
+ // TODO Also we need to have a list of table aliases to filter by primary or explicit partitions.
+ if (distributedJoins) {
+ boolean allCollocated = true;
+
+ for (GridCacheSqlQuery mapSqlQry : splitter.mapSqlQrys) {
+ Prepared prepared = optimize(h2, conn, mapSqlQry.query(), params, true, enforceJoinOrder);
+
+ allCollocated &= isCollocated((Query)prepared);
+
+ mapSqlQry.query(parse(prepared, true).getSQL());
+ }
+
+ // We do not need distributed joins if all MAP queries are colocated.
+ if (allCollocated)
+ distributedJoins = false;
+ }
+
+ // Setup resulting two step query and return it.
+ GridCacheTwoStepQuery twoStepQry = new GridCacheTwoStepQuery(originalSql, splitter.schemas, splitter.tbls);
+
+ twoStepQry.reduceQuery(splitter.rdcSqlQry);
+
+ for (GridCacheSqlQuery mapSqlQry : splitter.mapSqlQrys)
+ twoStepQry.addMapQuery(mapSqlQry);
+
+ twoStepQry.skipMergeTable(splitter.rdcQrySimple);
+ twoStepQry.explain(explain);
+ twoStepQry.distributedJoins(distributedJoins);
+
+ return twoStepQry;
+ }
+
+ /**
+ * @param qry Optimized and normalized query to split.
+ */
+ private void splitQuery(GridSqlQuery qry) {
+ // Create a fake parent AST element for the query to allow replacing the query in the parent by split.
+ GridSqlSubquery fakeQryPrnt = new GridSqlSubquery(qry);
+
+ // Fake parent query model. We need it just for convenience because buildQueryModel needs parent model.
+ QueryModel fakeQrymPrnt = new QueryModel(null, null, -1, null);
+
+ // Build a simplified query model. We need it because navigation over the original AST is too complex.
+ buildQueryModel(fakeQrymPrnt, fakeQryPrnt, 0, null);
+
+ assert fakeQrymPrnt.size() == 1;
+
+ // Get the built query model from the fake parent.
+ QueryModel qrym = fakeQrymPrnt.get(0);
+
+ // Setup the needed information for split.
+ analyzeQueryModel(qrym);
+
+ // If we have child queries to split, then go hard way.
+ if (qrym.needSplitChild) {
+ // All the siblings to selects we are going to split must be also wrapped into subqueries.
+ pushDownQueryModel(qrym);
+
+ // Need to make all the joined subqueries to be ordered by join conditions.
+ setupMergeJoinSorting(qrym);
+ }
+ else if (!qrym.needSplit) // Just split the top level query.
+ setNeedSplit(qrym);
+
+ // Split the query model into multiple map queries and a single reduce query.
+ splitQueryModel(qrym);
+
+ // Get back the updated query from the fake parent. It will be our reduce query.
+ qry = fakeQryPrnt.subquery();
+
+ // Setup a resulting reduce query.
+ rdcSqlQry = new GridCacheSqlQuery(qry.getSQL());
+ rdcQrySimple = qry.simpleQuery();
+
+ setupParameters(rdcSqlQry, qry, params);
+ }
+
+ /**
+ * @param qrym Query model.
+ */
+ private void pushDownQueryModel(QueryModel qrym) {
+ if (qrym.type == Type.UNION) {
+ assert qrym.needSplitChild; // Otherwise we should not get here.
+
+ for (int i = 0; i < qrym.size(); i++)
+ pushDownQueryModel(qrym.get(i));
+ }
+ else if (qrym.type == Type.SELECT) {
+ // If we already need to split, then no need to push down here.
+ if (!qrym.needSplit) {
+ assert qrym.needSplitChild; // Otherwise we should not get here.
+
+ pushDownQueryModelSelect(qrym);
+ }
+ }
+ else
+ throw new IllegalStateException("Type: " + qrym.type);
+ }
+
+ /**
+ * @param qrym Query model for the SELECT.
+ */
+ private void pushDownQueryModelSelect(QueryModel qrym) {
+ int begin = -1;
+
+ // Here we iterate over joined FROM table filters.
+ for (int i = 0; i < qrym.size(); i++) {
+ QueryModel child = qrym.get(i);
+
+ if (child.isQuery() && (child.needSplitChild || child.needSplit)) {
+ // Push down the currently collected range.
+ if (begin != -1) {
+ pushDownQueryModelRange(qrym, begin, i - 1);
+
+ i = begin + 1; // We've modified qrym by this range push down, need to adjust counter.
+
+ assert qrym.get(i) == child; // Adjustment check: we have to return to the same point.
+
+ // Reset range begin.
+ begin = -1;
+ }
+
+ if (child.needSplitChild)
+ pushDownQueryModel(child);
+ }
+ // It is a table or a function or a subquery that we do not need to split or split child.
+ // We need to find all the joined elements like this in chain to push them down.
+ else if (begin == -1)
+ begin = i;
+ }
+
+ // Push down the remaining range.
+ if (begin != -1)
+ pushDownQueryModelRange(qrym, begin, qrym.size() - 1);
+ }
+
+ /**
+ * @param qrym Query model.
+ */
+ private void setupMergeJoinSorting(QueryModel qrym) {
+ if (qrym.type == Type.UNION) {
+ for (int i = 0; i < qrym.size(); i++)
+ setupMergeJoinSorting(qrym.get(i));
+ }
+ else if (qrym.type == Type.SELECT) {
+ if (!qrym.needSplit) {
+ boolean needSplitChild = false;
+
+ for (int i = 0; i < qrym.size(); i++) {
+ QueryModel child = qrym.get(i);
+
+ assert child.isQuery() : qrym.type;
+
+ if (child.needSplit)
+ needSplitChild = true;
+ else
+ setupMergeJoinSorting(child); // Go deeper.
+ }
+
+ if (needSplitChild && qrym.size() > 1)
+ setupMergeJoinSortingSelect(qrym); // Setup merge join hierarchy for the SELECT.
+ }
+ }
+ else
+ throw new IllegalStateException("Type: " + qrym.type);
+ }
+
+ /**
+ * @param qrym Query model.
+ */
+ private void setupMergeJoinSortingSelect(QueryModel qrym) {
+ // After pushing down we can have the following variants:
+ // - splittable SELECT
+ // - subquery with splittable child (including UNION)
+
+ // We will push down chains of sequential subqueries between
+ // the splittable SELECT like in "push down" and
+ // setup sorting for all of them by joined fields.
+
+ for (int i = 1; i < qrym.size(); i++) {
+ QueryModel child = qrym.get(i);
+
+ if (child.needSplit) {
+ if (i > 1) {
+ // If we have multiple joined non-splittable subqueries before, then
+ // push down them into a single subquery.
+ doPushDownQueryModelRange(qrym, 0, i - 1, false);
+
+ i = 1;
+
+ assert qrym.get(i) == child; // We must remain at the same position.
+ }
+
+ injectSortingFirstJoin(qrym);
+ }
+ }
+ }
+
+ /**
+ * @param qrym Query model.
+ */
+ private void injectSortingFirstJoin(QueryModel qrym) {
+ // Must always be generated unique aliases here.
+ GridSqlAlias leftTbl = qrym.get(0).ast();
+ GridSqlAlias rightTbl = qrym.get(1).ast();
+
+ GridSqlJoin join = findJoin(qrym, 0);
+
+ // We are at the beginning, thus left and right AST must be children of the same join AST.
+ assert join.leftTable() == leftTbl: join.getSQL();
+ assert join.rightTable() == rightTbl: join.getSQL();
+
+ // Collect all AND conditions.
+ List<AndCondition> andConditions = new ArrayList<>();
+
+ collectAndConditions(andConditions, join, ON_CHILD);
+ collectAndConditions(andConditions, qrym.ast(), WHERE_CHILD);
+
+ // Collect all the JOIN condition columns for sorting.
+ List<GridSqlColumn> leftOrder = new ArrayList<>();
+ List<GridSqlColumn> rightOrder = new ArrayList<>();
+
+ for (int i = 0; i < andConditions.size(); i++) {
+ AndCondition and = andConditions.get(i);
+ GridSqlOperation op = and.ast();
+
+ if (op.operationType() == GridSqlOperationType.EQUAL) {
+ GridSqlAst leftExpr = op.child(0);
+ GridSqlAst rightExpr = op.child(1);
+
+ if (leftExpr instanceof GridSqlColumn && rightExpr instanceof GridSqlColumn) {
+ GridSqlAst leftFrom = ((GridSqlColumn)leftExpr).expressionInFrom();
+ GridSqlAst rightFrom = ((GridSqlColumn)rightExpr).expressionInFrom();
+
+ if (leftFrom == leftTbl && rightFrom == rightTbl) {
+ leftOrder.add((GridSqlColumn)leftExpr);
+ rightOrder.add((GridSqlColumn)rightExpr);
+ }
+ else if (leftFrom == rightTbl && rightFrom == leftTbl) {
+ leftOrder.add((GridSqlColumn)rightExpr);
+ rightOrder.add((GridSqlColumn)leftExpr);
+ }
+ }
+ }
+ }
+
+ // Do inject ORDER BY to the both sides.
+ injectOrderBy(leftTbl, leftOrder);
+ injectOrderBy(rightTbl, rightOrder);
+ }
+
+ /**
+ * @param subQryAlias Subquery alias.
+ * @param orderByCols Columns for ORDER BY.
+ */
+ private void injectOrderBy(GridSqlAlias subQryAlias, List<GridSqlColumn> orderByCols) {
+ // Structure: alias -> subquery -> query
+ GridSqlQuery qry = subQryAlias.child().child();
+ GridSqlSelect select = leftmostSelect(qry); // The leftmost SELECT in UNION defines column names.
+
+ for (int i = 0; i < orderByCols.size(); i++) {
+ GridSqlColumn col = orderByCols.get(i);
+
+ int colIdx = 0;
+
+ for (;;) {
+ GridSqlAst expr = select.column(colIdx);
+
+ String colName;
+
+ if (expr instanceof GridSqlAlias)
+ colName = ((GridSqlAlias)expr).alias();
+ else if (expr instanceof GridSqlColumn)
+ colName = ((GridSqlColumn)expr).columnName();
+ else
+ throw new IllegalStateException(); // It must be impossible to join by this column then.
+
+ if (colName.equals(col.columnName()))
+ break; // Found the needed column index.
+ }
+
+ // Add sort column to the query.
+ qry.addSort(new GridSqlSortColumn(colIdx, true, false, false));
+ }
}
/**
* @param qry Query.
- * @return Leftest simple query if this is UNION.
+ * @return Leftmost select if it is a UNION query.
*/
- private static GridSqlSelect leftest(GridSqlQuery qry) {
- if (qry instanceof GridSqlUnion)
- return leftest(((GridSqlUnion)qry).left());
+ private GridSqlSelect leftmostSelect(GridSqlQuery qry) {
+ while (qry instanceof GridSqlUnion)
+ qry = ((GridSqlUnion)qry).left();
return (GridSqlSelect)qry;
}
/**
- * @param qry Query.
- * @return Select query.
+ * @param qrym Query model.
+ * @param begin The first child model in range to push down.
+ * @param end The last child model in range to push down.
*/
- private static GridSqlSelect wrapUnion(GridSqlQuery qry) {
- if (qry instanceof GridSqlSelect)
- return (GridSqlSelect)qry;
+ private void pushDownQueryModelRange(QueryModel qrym, int begin, int end) {
+ assert end >= begin;
- // Handle UNION.
- GridSqlSelect wrapQry = new GridSqlSelect().from(new GridSqlSubquery(qry));
+ if (begin == end && qrym.get(end).isQuery()) {
+ // Simple case when we have a single subquery to push down, just mark it to be splittable.
+ setNeedSplit(qrym.get(end));
+ }
+ else {
+ // Here we have to generate a subquery for all the joined elements and
+ // and mark that subquery as splittable.
+ doPushDownQueryModelRange(qrym, begin, end, true);
+ }
+ }
- wrapQry.explain(qry.explain());
- qry.explain(false);
+ /**
+ * @param qrym Query model.
+ */
+ private void setNeedSplit(QueryModel qrym) {
+ if (qrym.type == Type.SELECT)
+ qrym.needSplit = true;
+ else if (qrym.type == Type.UNION) {
+ qrym.needSplitChild = true;
- GridSqlSelect left = leftest(qry);
+ // Mark all the selects in the UNION to be splittable.
+ for (QueryModel s : qrym) {
+ assert s.type == Type.SELECT: s.type;
- int c = 0;
+ s.needSplit = true;
+ }
+ }
+ else
+ throw new IllegalStateException("Type: " + qrym.type);
+ }
- for (GridSqlElement expr : left.columns(true)) {
- GridSqlType type = expr.resultType();
+ /**
+ * @param qrym Query model.
+ * @param begin The first child model in range to push down.
+ * @param end The last child model in range to push down.
+ * @param needSplit If we will need to split the created subquery model.
+ */
+ private void doPushDownQueryModelRange(QueryModel qrym, int begin, int end, boolean needSplit) {
+ // Create wrap query where we will push all the needed joined elements, columns and conditions.
+ GridSqlSelect wrapSelect = new GridSqlSelect();
+ GridSqlSubquery wrapSubqry = new GridSqlSubquery(wrapSelect);
+ GridSqlAlias wrapAlias = alias(nextUniqueTableAlias(null), wrapSubqry);
- String colName;
+ QueryModel wrapQrym = new QueryModel(Type.SELECT, wrapSubqry, 0, wrapAlias);
- if (expr instanceof GridSqlAlias)
- colName = ((GridSqlAlias)expr).alias();
- else if (expr instanceof GridSqlColumn)
- colName = ((GridSqlColumn)expr).columnName();
- else {
- colName = columnName(c);
+ wrapQrym.needSplit = needSplit;
- expr = alias(colName, expr);
+ // Prepare all the prerequisites.
+ GridSqlSelect select = qrym.ast();
- // Set generated alias to the expression.
- left.setColumn(c, expr);
- }
+ Set<GridSqlAlias> tblAliases = newIdentityHashSet();
+ Map<String,GridSqlAlias> cols = new HashMap<>();
+
+ // Collect all the tables for push down.
+ for (int i = begin; i <= end; i++) {
+ GridSqlAlias uniqueTblAlias = qrym.get(i).uniqueAlias;
+
+ assert uniqueTblAlias != null: select.getSQL();
+
+ tblAliases.add(uniqueTblAlias);
+ }
+
+ // Push down columns in SELECT clause.
+ pushDownSelectColumns(tblAliases, cols, wrapAlias, select);
+
+ // Move all the related WHERE conditions to wrap query.
+ pushDownWhereConditions(tblAliases, cols, wrapAlias, select);
+
+ // Push down to a subquery all the JOIN elements and process ON conditions.
+ pushDownJoins(tblAliases, cols, qrym, begin, end, wrapAlias);
+
+ // Add all the collected columns to the wrap query.
+ for (GridSqlAlias col : cols.values())
+ wrapSelect.addColumn(col, true);
- GridSqlColumn col = column(colName);
+ // Adjust query models to a new AST structure.
- col.resultType(type);
+ // Move pushed down child models to the newly created model.
+ for (int i = begin; i <= end; i++) {
+ QueryModel child = qrym.get(i);
- wrapQry.addColumn(col, true);
+ assert !child.needSplit && !child.needSplitChild;
- c++;
+ wrapQrym.add(child);
}
- // ORDER BY
- if (!qry.sort().isEmpty()) {
- for (GridSqlSortColumn col : qry.sort())
- wrapQry.addSort(col);
+ // Replace the first child model with the created one.
+ qrym.set(begin, wrapQrym);
+
+ // Drop others.
+ for (int x = begin + 1, i = x; i <= end; i++)
+ qrym.remove(x);
+ }
+
+ /**
+ * @param tblAliases Table aliases for push down.
+ * @param cols Columns with generated aliases.
+ * @param qrym Query model.
+ * @param begin The first child model in range to push down.
+ * @param end The last child model in range to push down.
+ * @param wrapAlias Alias of the wrap query.
+ */
+ private void pushDownJoins(
+ Set<GridSqlAlias> tblAliases,
+ Map<String,GridSqlAlias> cols,
+ QueryModel qrym,
+ int begin,
+ int end,
+ GridSqlAlias wrapAlias
+ ) {
+ GridSqlSelect wrapSelect = wrapAlias.child();
+
+ final int last = qrym.size() - 1;
+
+ if (begin == end) {
+ // Simple case when we have to push down only a single table and no joins.
+
+ // join3
+ // / \
+ // join2 \
+ // / \ \
+ // join1 \ \
+ // / \ \ \
+ // T0 T1 T2 T3
+ // ^^
+
+ // - Push down T2 to the wrap query W and replace T2 with W:
+
+ // join3
+ // / \
+ // join2 \
+ // / \ \
+ // join1 \ \
+ // / \ \ \
+ // T0 T1 W T3
+
+ // W: T2
+
+ GridSqlJoin endJoin = findJoin(qrym, end);
+
+ wrapSelect.from(qrym.get(end).uniqueAlias);
+ endJoin.child(end == 0 ? LEFT_TABLE_CHILD : RIGHT_TABLE_CHILD, wrapAlias);
+ }
+ else if (end == last) {
+ // Here we need to push down chain from `begin` to the last child in the model (T3).
+ // Thus we have something to split in the beginning and `begin` can not be 0.
+ assert begin > 0;
+
+ // join3
+ // / \
+ // join2 \
+ // / \ \
+ // join1 \ \
+ // / \ \ \
+ // T0 T1 T2 T3
+ // ^----^
+
+ // We have to push down T2 and T3.
+ // Also may be T1, but this does not change much the logic.
+
+ // - Add join3 to W,
+ // - replace left branch in join3 with T2,
+ // - replace right branch in join2 with W:
+
+ // join2
+ // / \
+ // join1 \
+ // / \ \
+ // T0 T1 W
+
+ // W: join3
+ // / \
+ // T2 T3
+
+ GridSqlJoin beginJoin = findJoin(qrym, begin);
+ GridSqlJoin afterBeginJoin = findJoin(qrym, begin + 1);
+ GridSqlJoin endJoin = findJoin(qrym, end);
+
+ wrapSelect.from(endJoin);
+ afterBeginJoin.leftTable(beginJoin.rightTable());
+ beginJoin.rightTable(wrapAlias);
+ }
+ else if (begin == 0) {
+ // From the first model to some middle one.
+
+ // join3
+ // / \
+ // join2 \
+ // / \ \
+ // join1 \ \
+ // / \ \ \
+ // T0 T1 T2 T3
+ // ^-----^
+
+
+ // join3
+ // / \
+ // join2 \
+ // / \ \
+ // W T2 T3
+
+ // W: join1
+ // / \
+ // T0 T1
+
+ GridSqlJoin endJoin = findJoin(qrym, end);
+ GridSqlJoin afterEndJoin = findJoin(qrym, end + 1);
+
+ wrapSelect.from(endJoin);
+ afterEndJoin.leftTable(wrapAlias);
+ }
+ else {
+ // Push down some middle range.
+
+ // join3
+ // / \
+ // join2 \
+ // / \ \
+ // join1 \ \
+ // / \ \ \
+ // T0 T1 T2 T3
+ // ^----^
+
+
+ // join3
+ // / \
+ // join1 \
+ // / \ \
+ // T0 W T3
+
+ // W: join2
+ // / \
+ // T1 T2
+
+ GridSqlJoin beginJoin = findJoin(qrym, begin);
+ GridSqlJoin afterBeginJoin = findJoin(qrym, begin + 1);
+ GridSqlJoin endJoin = findJoin(qrym, end);
+ GridSqlJoin afterEndJoin = findJoin(qrym, end + 1);
+
+ wrapSelect.from(endJoin);
+ afterEndJoin.leftTable(beginJoin);
+ afterBeginJoin.leftTable(beginJoin.rightTable());
+ beginJoin.rightTable(wrapAlias);
}
- return wrapQry;
+ // Get the original SELECT.
+ GridSqlSelect select = qrym.ast();
+ GridSqlAst from = select.from();
+
+ // Push down related ON conditions for all the related joins.
+ while (from instanceof GridSqlJoin) {
+ pushDownColumnsInExpression(tblAliases, cols, wrapAlias, from, ON_CHILD);
+
+ from = from.child(LEFT_TABLE_CHILD);
+ }
}
/**
- * @param stmt Prepared statement.
- * @param params Parameters.
- * @param collocatedGrpBy Whether the query has collocated GROUP BY keys.
- * @param distributedJoins If distributed joins enabled.
- * @return Two step query.
+ * @param tblAliases Table aliases for push down.
+ * @param cols Columns with generated aliases.
+ * @param wrapAlias Alias of the wrap query.
+ * @param select The original select.
*/
- public static GridCacheTwoStepQuery split(
- JdbcPreparedStatement stmt,
- Object[] params,
- final boolean collocatedGrpBy,
- final boolean distributedJoins
+ private void pushDownSelectColumns(
+ Set<GridSqlAlias> tblAliases,
+ Map<String,GridSqlAlias> cols,
+ GridSqlAlias wrapAlias,
+ GridSqlSelect select
) {
- if (params == null)
- params = GridCacheSqlQuery.EMPTY_PARAMS;
+ for (int i = 0; i < select.allColumns(); i++) {
+ GridSqlAst expr = select.column(i);
+
+ if (expr instanceof GridSqlColumn) {
+ // If this is a column with no expression and without alias, we need replace it with an alias,
+ // because in the wrapQuery we will generate unique aliases for all the columns to avoid duplicates
+ // and all the columns in the will be replaced.
+ expr = alias(((GridSqlColumn)expr).columnName(), expr);
+
+ select.setColumn(i, expr);
+ }
+
+ for (int c = 0; c < expr.size(); c++)
+ pushDownColumnsInExpression(tblAliases, cols, wrapAlias, expr, c);
+ }
+ }
+
+ /**
+ * @param tblAliases Table aliases to push down.
+ * @param cols Columns with generated aliases.
+ * @param wrapAlias Alias of the wrap query.
+ * @param prnt Parent expression.
+ * @param childIdx Child index.
+ */
+ @SuppressWarnings("SuspiciousMethodCalls")
+ private void pushDownColumnsInExpression(
+ Set<GridSqlAlias> tblAliases,
+ Map<String,GridSqlAlias> cols,
+ GridSqlAlias wrapAlias,
+ GridSqlAst prnt,
+ int childIdx
+ ) {
+ GridSqlAst child = prnt.child(childIdx);
- Set<String> tbls = new HashSet<>();
- Set<String> schemas = new HashSet<>();
+ if (child instanceof GridSqlColumn)
+ pushDownColumn(tblAliases, cols, wrapAlias, prnt, childIdx);
+ else {
+ for (int i = 0; i < child.size(); i++)
+ pushDownColumnsInExpression(tblAliases, cols, wrapAlias, child, i);
+ }
+ }
- final Prepared prepared = prepared(stmt);
+ /**
+ * @param tblAliases Table aliases for push down.
+ * @param cols Columns with generated aliases.
+ * @param wrapAlias Alias of the wrap query.
+ * @param prnt Parent element.
+ * @param childIdx Child index.
+ */
+ private void pushDownColumn(
+ Set<GridSqlAlias> tblAliases,
+ Map<String,GridSqlAlias> cols,
+ GridSqlAlias wrapAlias,
+ GridSqlAst prnt,
+ int childIdx
+ ) {
+ GridSqlColumn col = prnt.child(childIdx);
- GridSqlStatement gridStmt = new GridSqlQueryParser().parse(prepared);
+ // It must always be unique table alias.
+ GridSqlAlias tblAlias = (GridSqlAlias)col.expressionInFrom();
- assert gridStmt instanceof GridSqlQuery;
+ assert tblAlias != null; // The query is normalized.
- GridSqlQuery qry = (GridSqlQuery) gridStmt;
+ if (!tblAliases.contains(tblAlias))
+ return;
- qry = collectAllTables(qry, schemas, tbls);
+ String uniqueColAlias = uniqueColumnAlias(col);
+ GridSqlAlias colAlias = cols.get(uniqueColAlias);
- // Build resulting two step query.
- GridCacheTwoStepQuery res = new GridCacheTwoStepQuery(qry.getSQL(), schemas, tbls);
+ if (colAlias == null) {
+ colAlias = alias(uniqueColAlias, col);
- // Map query will be direct reference to the original query AST.
- // Thus all the modifications will be performed on the original AST, so we should be careful when
- // nullifying or updating things, have to make sure that we will not need them in the original form later.
- final GridSqlSelect mapQry = wrapUnion(qry);
+ // We have this map to avoid column duplicates in wrap query.
+ cols.put(uniqueColAlias, colAlias);
+ }
- GridCacheSqlQuery rdc = split(res, 0, mapQry, params, collocatedGrpBy);
+ col = column(uniqueColAlias);
+ // col.tableAlias(wrapAlias.alias());
+ col.expressionInFrom(wrapAlias);
- res.reduceQuery(rdc);
+ prnt.child(childIdx, col);
+ }
- // We do not have to look at each map query separately here, because if
- // the whole initial query is collocated, then all the map sub-queries
- // will be collocated as well.
- res.distributedJoins(distributedJoins && !isCollocated(query(prepared)));
+ /**
+ * @param col Column.
+ * @return Unique column alias based on generated unique table alias.
+ */
+ private String uniqueColumnAlias(GridSqlColumn col) {
+ GridSqlAlias uniqueTblAlias = (GridSqlAlias)col.expressionInFrom();
- return res;
+ return uniqueTblAlias.alias() + "__" + col.columnName();
}
/**
- * @param el Either {@link GridSqlSelect#from()} or {@link GridSqlSelect#where()} elements.
+ * @param tblAliases Table aliases for push down.
+ * @param cols Columns with generated aliases.
+ * @param wrapAlias Alias of the wrap query.
+ * @param select The original select.
*/
- private static void findAffinityColumnConditions(GridSqlElement el) {
- if (el == null)
+ private void pushDownWhereConditions(
+ Set<GridSqlAlias> tblAliases,
+ Map<String,GridSqlAlias> cols,
+ GridSqlAlias wrapAlias,
+ GridSqlSelect select
+ ) {
+ if (select.where() == null)
return;
- el = GridSqlAlias.unwrap(el);
+ GridSqlSelect wrapSelect = wrapAlias.child();
- if (el instanceof GridSqlJoin) {
- GridSqlJoin join = (GridSqlJoin)el;
+ List<AndCondition> andConditions = new ArrayList<>();
- findAffinityColumnConditions(join.leftTable());
- findAffinityColumnConditions(join.rightTable());
- findAffinityColumnConditions(join.on());
+ collectAndConditions(andConditions, select, WHERE_CHILD);
+
+ for (int i = 0; i < andConditions.size(); i++) {
+ AndCondition c = andConditions.get(i);
+ GridSqlAst condition = c.ast();
+
+ if (isAllRelatedToTables(tblAliases, condition)) {
+ if (!isTrue(condition)) {
+ // Replace the original condition with `true` and move it to the wrap query.
+ c.prnt.child(c.childIdx, TRUE);
+ wrapSelect.whereAnd(condition);
+ }
+ }
+ else
+ pushDownColumnsInExpression(tblAliases, cols, wrapAlias, c.prnt, c.childIdx);
}
- else if (el instanceof GridSqlOperation) {
- GridSqlOperationType type = ((GridSqlOperation)el).operationType();
+ }
- switch(type) {
- case AND:
- findAffinityColumnConditions(el.child(0));
- findAffinityColumnConditions(el.child(1));
+ /**
+ * @param expr Expression.
+ * @return {@code true} If this expression represents a constant value `TRUE`.
+ */
+ private static boolean isTrue(GridSqlAst expr) {
+ return expr instanceof GridSqlConst && ((GridSqlConst)expr).value() == TRUE.value();
+ }
- break;
+ /**
+ * @param tblAliases Table aliases for push down.
+ * @param ast AST.
+ * @return {@code true} If all the columns in the given expression are related to the given tables.
+ */
+ @SuppressWarnings("SuspiciousMethodCalls")
+ private boolean isAllRelatedToTables(Set<GridSqlAlias> tblAliases, GridSqlAst ast) {
+ if (ast instanceof GridSqlColumn) {
+ GridSqlColumn col = (GridSqlColumn)ast;
- case EQUAL:
- findAffinityColumn(el.child(0));
- findAffinityColumn(el.child(1));
+ if (!tblAliases.contains(col.expressionInFrom()))
+ return false;
+ }
+ else {
+ for (int i = 0; i < ast.size(); i++) {
+ if (!isAllRelatedToTables(tblAliases, ast))
+ return false;
}
}
+
+ return true;
}
/**
- * @param exp Possible affinity column expression.
+ * @param andConditions Conditions in AND.
+ * @param prnt Parent Parent element.
+ * @param childIdx Child index.
*/
- private static void findAffinityColumn(GridSqlElement exp) {
- if (exp instanceof GridSqlColumn) {
- GridSqlColumn col = (GridSqlColumn)exp;
+ private void collectAndConditions(List<AndCondition> andConditions, GridSqlAst prnt, int childIdx) {
+ GridSqlAst child = prnt.child(childIdx);
- GridSqlElement from = col.expressionInFrom();
+ if (child instanceof GridSqlOperation) {
+ GridSqlOperation op = (GridSqlOperation)child;
- if (from instanceof GridSqlTable) {
- GridSqlTable fromTbl = (GridSqlTable)from;
+ if (op.operationType() == GridSqlOperationType.AND) {
+ collectAndConditions(andConditions, op, 0);
+ collectAndConditions(andConditions, op, 1);
- GridH2Table tbl = fromTbl.dataTable();
+ return;
+ }
+ }
- if (tbl != null) {
- IndexColumn affKeyCol = tbl.getAffinityKeyColumn();
- Column expCol = col.column();
+ if (!isTrue(child))
+ andConditions.add(new AndCondition(prnt, childIdx));
+ }
- if (affKeyCol != null && expCol != null &&
- affKeyCol.column.getColumnId() == expCol.getColumnId()) {
- // Mark that table lookup will use affinity key.
- fromTbl.affinityKeyCondition(true);
- }
+ /**
+ * @return New identity hash set.
+ */
+ private static <X> Set<X> newIdentityHashSet() {
+ return Collections.newSetFromMap(new IdentityHashMap<X,Boolean>());
+ }
+
+ /**
+ * @param qrym Query model for the SELECT.
+ * @param idx Index of the child model for which we need to find a respective JOIN element.
+ * @return JOIN.
+ */
+ private static GridSqlJoin findJoin(QueryModel qrym, int idx) {
+ assert qrym.type == Type.SELECT: qrym.type;
+ assert qrym.size() > 1; // It must be at least one join with at least two child tables.
+ assert idx < qrym.size(): idx;
+
+ // join2
+ // / \
+ // join1 \
+ // / \ \
+ // T0 T1 T2
+
+ // If we need to find JOIN for T0, it is the same as for T1.
+ if (idx == 0)
+ idx = 1;
+
+ GridSqlJoin join = (GridSqlJoin)qrym.<GridSqlSelect>ast().from();
+
+ for (int i = qrym.size() - 1; i > idx; i--)
+ join = (GridSqlJoin)join.leftTable();
+
+ assert join.rightTable() == qrym.get(idx).ast();
+
+ return join;
+ }
+
+ /**
+ * @param qrym Query model.
+ */
+ private void splitQueryModel(QueryModel qrym) {
+ switch (qrym.type) {
+ case SELECT:
+ if (qrym.needSplit) {
+ splitSelect(qrym.prnt, qrym.childIdx);
+
+ break;
}
- }
+
+ // Intentional fallthrough to go deeper.
+ case UNION:
+ for (int i = 0; i < qrym.size(); i++)
+ splitQueryModel(qrym.get(i));
+
+ break;
+
+ default:
+ throw new IllegalStateException("Type: " + qrym.type);
}
}
/**
- * @param qry Select.
- * @return {@code true} If there is at least one partitioned table in FROM clause.
+ * @param qrym Query model.
*/
- private static boolean hasPartitionedTableInFrom(GridSqlSelect qry) {
- return findTablesInFrom(qry.from(), new IgnitePredicate<GridSqlElement>() {
- @Override public boolean apply(GridSqlElement el) {
- if (el instanceof GridSqlTable) {
- GridH2Table tbl = ((GridSqlTable)el).dataTable();
+ private void analyzeQueryModel(QueryModel qrym) {
+ if (!qrym.isQuery())
+ return;
+
+ // Process all the children at the beginning: depth first analysis.
+ for (int i = 0; i < qrym.size(); i++) {
+ QueryModel child = qrym.get(i);
- assert tbl != null : el;
+ analyzeQueryModel(child);
- GridCacheContext<?,?> cctx = tbl.rowDescriptor().context();
+ // Pull up information about the splitting child.
+ if (child.needSplit || child.needSplitChild)
+ qrym.needSplitChild = true; // We have a child to split.
+ }
- return !cctx.isLocal() && !cctx.isReplicated();
+ if (qrym.type == Type.SELECT) {
+ // We may need to split the SELECT only if it has no splittable children,
+ // because only the downmost child can be split, the parents will be the part of
+ // the reduce query.
+ if (!qrym.needSplitChild)
+ qrym.needSplit = needSplitSelect(qrym.<GridSqlSelect>ast()); // Only SELECT can have this flag in true.
+ }
+ else if (qrym.type == Type.UNION) {
+ // If it is not a UNION ALL, then we have to split because otherwise we can produce duplicates or
+ // wrong results for UNION DISTINCT, EXCEPT, INTERSECT queries.
+ if (!qrym.needSplitChild && !qrym.unionAll)
+ qrym.needSplitChild = true;
+
+ // If we have to split some child SELECT in this UNION, then we have to enforce split
+ // for all other united selects, because this UNION has to be a part of the reduce query,
+ // thus each SELECT needs to have a reduce part for this UNION, but the whole SELECT can not
+ // be a reduce part (usually).
+ if (qrym.needSplitChild) {
+ for (int i = 0; i < qrym.size(); i++) {
+ QueryModel child = qrym.get(i);
+
+ assert child.type == Type.SELECT : child.type;
+
+ if (!child.needSplitChild && !child.needSplit)
+ child.needSplit = true;
}
+ }
+ }
+ else
+ throw new IllegalStateException("Type: " + qrym.type);
+ }
- return false;
+ /**
+ * @param prntModel Parent model.
+ * @param prnt Parent AST element.
+ * @param childIdx Child index.
+ * @param uniqueAlias Unique parent alias of the current element.
+ */
+ private void buildQueryModel(QueryModel prntModel, GridSqlAst prnt, int childIdx, GridSqlAlias uniqueAlias) {
+ GridSqlAst child = prnt.child(childIdx);
+
+ assert child != null;
+
+ if (child instanceof GridSqlSelect) {
+ QueryModel model = new QueryModel(Type.SELECT, prnt, childIdx, uniqueAlias);
+
+ prntModel.add(model);
+
+ buildQueryModel(model, child, FROM_CHILD, null);
+ }
+ else if (child instanceof GridSqlUnion) {
+ QueryModel model;
+
+ // We will collect all the selects into a single UNION model.
+ if (prntModel.type == Type.UNION)
+ model = prntModel;
+ else {
+ model = new QueryModel(Type.UNION, prnt, childIdx, uniqueAlias);
+
+ prntModel.add(model);
}
- });
+
+ if (((GridSqlUnion)child).unionType() != SelectUnion.UNION_ALL)
+ model.unionAll = false;
+
+ buildQueryModel(model, child, LEFT_CHILD, null);
+ buildQueryModel(model, child, RIGHT_CHILD, null);
+ }
+ else {
+ // Here we must be in FROM clause of the SELECT.
+ assert prntModel.type == Type.SELECT : prntModel.type;
+
+ if (child instanceof GridSqlAlias)
+ buildQueryModel(prntModel, child, 0, (GridSqlAlias)child);
+ else if (child instanceof GridSqlJoin) {
+ buildQueryModel(prntModel, child, LEFT_TABLE_CHILD, uniqueAlias);
+ buildQueryModel(prntModel, child, RIGHT_TABLE_CHILD, uniqueAlias);
+ }
+ else {
+ // Here we must be inside of generated unique alias for FROM clause element.
+ assert prnt == uniqueAlias: prnt.getClass();
+
+ if (child instanceof GridSqlTable)
+ prntModel.add(new QueryModel(Type.TABLE, prnt, childIdx, uniqueAlias));
+ else if (child instanceof GridSqlSubquery)
+ buildQueryModel(prntModel, child, 0, uniqueAlias);
+ else if (child instanceof GridSqlFunction)
+ prntModel.add(new QueryModel(Type.FUNCTION, prnt, childIdx, uniqueAlias));
+ else
+ throw new IllegalStateException("Unknown child type: " + child.getClass());
+ }
+ }
}
/**
- * @param res Resulting two step query.
- * @param splitIdx Split index.
- * @param mapQry Map query to be split.
- * @param params Query parameters.
- * @param collocatedGroupBy Whether the query has collocated GROUP BY keys.
- * @return Reduce query for the given map query.
+ * @param select Select to check.
+ * @return {@code true} If we need to split this select.
*/
- private static GridCacheSqlQuery split(GridCacheTwoStepQuery res, int splitIdx, final GridSqlSelect mapQry,
- Object[] params, boolean collocatedGroupBy) {
- final boolean explain = mapQry.explain();
+ private boolean needSplitSelect(GridSqlSelect select) {
+ if (select.distinct())
+ return true;
- mapQry.explain(false);
+ if (collocatedGrpBy)
+ return false;
- GridSqlSelect rdcQry = new GridSqlSelect().from(table(splitIdx));
+ if (select.groupColumns() != null)
+ return true;
- // Split all select expressions into map-reduce parts.
- List<GridSqlElement> mapExps = new ArrayList<>(mapQry.allColumns());
+ for (int i = 0; i < select.allColumns(); i++) {
+ if (hasAggregates(select.column(i)))
+ return true;
+ }
- mapExps.addAll(mapQry.columns(false));
+ return false;
+ }
+
+ /**
+ * !!! Notice that here we will modify the original query AST in this method.
+ *
+ * @param prnt Parent AST element.
+ * @param childIdx Index of child select.
+ */
+ private void splitSelect(
+ final GridSqlAst prnt,
+ final int childIdx
+ ) {
+ if (++splitId > 99)
+ throw new CacheException("Too complex query to process.");
+
+ final GridSqlSelect mapQry = prnt.child(childIdx);
final int visibleCols = mapQry.visibleColumns();
- final int havingCol = mapQry.havingColumn();
- List<GridSqlElement> rdcExps = new ArrayList<>(visibleCols);
+ List<GridSqlAst> rdcExps = new ArrayList<>(visibleCols);
+ List<GridSqlAst> mapExps = new ArrayList<>(mapQry.allColumns());
+
+ mapExps.addAll(mapQry.columns(false));
Set<String> colNames = new HashSet<>();
+ final int havingCol = mapQry.havingColumn();
boolean distinctAggregateFound = false;
- if (!collocatedGroupBy) {
+ if (!collocatedGrpBy) {
for (int i = 0, len = mapExps.size(); i < len; i++)
distinctAggregateFound |= hasDistinctAggregates(mapExps.get(i));
}
boolean aggregateFound = distinctAggregateFound;
+ // Split all select expressions into map-reduce parts.
for (int i = 0, len = mapExps.size(); i < len; i++) // Remember len because mapExps list can grow.
- aggregateFound |= splitSelectExpression(mapExps, rdcExps, colNames, i, collocatedGroupBy, i == havingCol,
+ aggregateFound |= splitSelectExpression(mapExps, rdcExps, colNames, i, collocatedGrpBy, i == havingCol,
distinctAggregateFound);
+ assert !(collocatedGrpBy && aggregateFound); // We do not split aggregates when collocatedGrpBy is true.
+
+ // Create reduce query AST. Use unique merge table for this split.
+ GridSqlSelect rdcQry = new GridSqlSelect().from(mergeTable(splitId));
+
// -- SELECT
mapQry.clearColumns();
- for (GridSqlElement exp : mapExps) // Add all map expressions as visible.
+ for (GridSqlAst exp : mapExps) // Add all map expressions as visible.
mapQry.addColumn(exp, true);
for (int i = 0; i < visibleCols; i++) // Add visible reduce columns.
@@ -332,14 +1216,10 @@ public class GridSqlQuerySplitter {
for (int i = rdcExps.size(); i < mapExps.size(); i++) // Add all extra map columns as invisible reduce columns.
rdcQry.addColumn(column(((GridSqlAlias)mapExps.get(i)).alias()), false);
- // -- FROM
- findAffinityColumnConditions(mapQry.from());
-
- // -- WHERE
- findAffinityColumnConditions(mapQry.where());
+ // -- FROM WHERE: do nothing
// -- GROUP BY
- if (mapQry.groupColumns() != null && !collocatedGroupBy) {
+ if (mapQry.groupColumns() != null && !collocatedGrpBy) {
rdcQry.groupColumns(mapQry.groupColumns());
// Grouping with distinct aggregates cannot be performed on map phase
@@ -348,11 +1228,11 @@ public class GridSqlQuerySplitter {
}
// -- HAVING
- if (havingCol >= 0 && !collocatedGroupBy) {
+ if (havingCol >= 0 && !collocatedGrpBy) {
// TODO IGNITE-1140 - Find aggregate functions in HAVING clause or rewrite query to put all aggregates to SELECT clause.
// We need to find HAVING column in reduce query.
for (int i = visibleCols; i < rdcQry.allColumns(); i++) {
- GridSqlElement c = rdcQry.column(i);
+ GridSqlAst c = rdcQry.column(i);
if (c instanceof GridSqlAlias && HAVING_COLUMN.equals(((GridSqlAlias)c).alias())) {
rdcQry.havingColumn(i);
@@ -369,6 +1249,7 @@ public class GridSqlQuerySplitter {
for (GridSqlSortColumn sortCol : mapQry.sort())
rdcQry.addSort(sortCol);
+ // If collocatedGrpBy is true, then aggregateFound is always false.
if (aggregateFound) // Ordering over aggregates does not make sense.
mapQry.clearSort(); // Otherwise map sort will be used by offset-limit.
// TODO IGNITE-1141 - Check if sorting is done over aggregated expression, otherwise we can sort and use offset-limit.
@@ -378,6 +1259,8 @@ public class GridSqlQuerySplitter {
if (mapQry.limit() != null) {
rdcQry.limit(mapQry.limit());
+ // Will keep limits on map side when collocatedGrpBy is true,
+ // because in this case aggregateFound is always false.
if (aggregateFound)
mapQry.limit(null);
}
@@ -398,38 +1281,40 @@ public class GridSqlQuerySplitter {
rdcQry.distinct(true);
}
- IntArray paramIdxs = new IntArray(params.length);
+ // Replace the given select with generated reduce query in the parent.
+ prnt.child(childIdx, rdcQry);
- GridCacheSqlQuery map = new GridCacheSqlQuery(mapQry.getSQL(),
- findParams(mapQry, params, new ArrayList<>(params.length), paramIdxs).toArray());
+ // Setup resulting map query.
+ GridCacheSqlQuery map = new GridCacheSqlQuery(mapQry.getSQL());
+ setupParameters(map, mapQry, params);
map.columns(collectColumns(mapExps));
- map.parameterIndexes(toArray(paramIdxs));
-
- res.addMapQuery(map);
-
- res.explain(explain);
- paramIdxs = new IntArray(params.length);
+ mapSqlQrys.add(map);
+ }
- GridCacheSqlQuery rdc = new GridCacheSqlQuery(rdcQry.getSQL(),
- findParams(rdcQry, params, new ArrayList<>(), paramIdxs).toArray());
+ /**
+ * @param sqlQry Query.
+ * @param qryAst Select AST.
+ * @param params All parameters.
+ */
+ private static void setupParameters(GridCacheSqlQuery sqlQry, GridSqlQuery qryAst, Object[] params) {
+ IntArray paramIdxs = new IntArray(params.length);
- rdc.parameterIndexes(toArray(paramIdxs));
- res.skipMergeTable(rdcQry.simpleQuery());
+ params = findParams(qryAst, params, new ArrayList<>(params.length), paramIdxs).toArray();
- return rdc;
+ sqlQry.parameters(params, toArray(paramIdxs));
}
/**
* @param cols Columns from SELECT clause.
* @return Map of columns with types.
*/
- private static LinkedHashMap<String,?> collectColumns(List<GridSqlElement> cols) {
+ private LinkedHashMap<String,?> collectColumns(List<GridSqlAst> cols) {
LinkedHashMap<String, GridSqlType> res = new LinkedHashMap<>(cols.size(), 1f, false);
for (int i = 0; i < cols.size(); i++) {
- GridSqlElement col = cols.get(i);
+ GridSqlAst col = cols.get(i);
GridSqlType t = col.resultType();
if (t == null)
@@ -453,113 +1338,204 @@ public class GridSqlQuerySplitter {
}
/**
+ * @param prepared Prepared command.
+ * @param useOptimizedSubqry Use optimized subqueries order for table filters.
+ * @return Parsed SQL query AST.
+ */
+ private static GridSqlQuery parse(Prepared prepared, boolean useOptimizedSubqry) {
+ return (GridSqlQuery)new GridSqlQueryParser(useOptimizedSubqry).parse(prepared);
+ }
+
+ /**
+ * @param h2 Indexing.
+ * @param c Connection.
+ * @param qry Parsed query.
+ * @param params Query parameters.
+ * @param enforceJoinOrder Enforce join order.
+ * @return Optimized prepared command.
+ * @throws SQLException If failed.
+ * @throws IgniteCheckedException If failed.
+ */
+ private static Prepared optimize(
+ IgniteH2Indexing h2,
+ Connection c,
+ String qry,
+ Object[] params,
+ boolean distributedJoins,
+ boolean enforceJoinOrder
+ ) throws SQLException, IgniteCheckedException {
+ setupConnection(c, distributedJoins, enforceJoinOrder);
+
+ try (PreparedStatement s = c.prepareStatement(qry)) {
+ h2.bindParameters(s, F.asList(params));
+
+ return prepared(s);
+ }
+ }
+
+ /**
* @param qry Query.
- * @param schemas Schema names.
- * @param tbls Tables.
- * @return Query.
*/
- private static GridSqlQuery collectAllTables(GridSqlQuery qry, Set<String> schemas, Set<String> tbls) {
+ private void normalizeQuery(GridSqlQuery qry) {
if (qry instanceof GridSqlUnion) {
GridSqlUnion union = (GridSqlUnion)qry;
- collectAllTables(union.left(), schemas, tbls);
- collectAllTables(union.right(), schemas, tbls);
+ normalizeQuery(union.left());
+ normalizeQuery(union.right());
}
else {
GridSqlSelect select = (GridSqlSelect)qry;
- collectAllTablesInFrom(select.from(), schemas, tbls);
+ // Normalize FROM first to update column aliases after.
+ normalizeFrom(select, FROM_CHILD, false);
- for (GridSqlElement el : select.columns(false))
- collectAllTablesInSubqueries(el, schemas, tbls);
+ List<GridSqlAst> cols = select.columns(false);
- collectAllTablesInSubqueries(select.where(), schemas, tbls);
+ for (int i = 0; i < cols.size(); i++)
+ normalizeExpression(select, childIndexForColumn(i));
+
+ normalizeExpression(select, WHERE_CHILD);
+
+ // ORDER BY and HAVING are in SELECT expressions.
}
- return qry;
+ normalizeExpression(qry, OFFSET_CHILD);
+ normalizeExpression(qry, LIMIT_CHILD);
}
/**
- * @param from From element.
- * @param schemas Schema names.
- * @param tbls Tables.
+ * @param prnt Table parent element.
+ * @param childIdx Child index for the table or alias containing the table.
+ * @return Generated alias.
*/
- private static void collectAllTablesInFrom(GridSqlElement from, final Set<String> schemas, final Set<String> tbls) {
- findTablesInFrom(from, new IgnitePredicate<GridSqlElement>() {
- @Override public boolean apply(GridSqlElement el) {
- if (el instanceof GridSqlTable) {
- GridSqlTable tbl = (GridSqlTable)el;
+ private GridSqlAlias generateUniqueAlias(GridSqlAst prnt, int childIdx) {
+ GridSqlAst child = prnt.child(childIdx);
+ GridSqlAst tbl = GridSqlAlias.unwrap(child);
- String schema = tbl.schema();
+ assert tbl instanceof GridSqlTable || tbl instanceof GridSqlSubquery ||
+ tbl instanceof GridSqlFunction: tbl.getClass();
- boolean addSchema = tbls == null;
+ String uniqueAlias = nextUniqueTableAlias(tbl != child ? ((GridSqlAlias)child).alias() : null);
- if (tbls != null)
- addSchema = tbls.add(tbl.dataTable().identifier());
+ GridSqlAlias uniqueAliasAst = new GridSqlAlias(uniqueAlias, tbl);
- if (addSchema && schema != null && schemas != null)
- schemas.add(schema);
- }
- else if (el instanceof GridSqlSubquery)
- collectAllTables(((GridSqlSubquery)el).select(), schemas, tbls);
+ uniqueFromAliases.put(tbl, uniqueAliasAst);
- return false;
- }
- });
+ // Replace the child in the parent.
+ prnt.child(childIdx, uniqueAliasAst);
+
+ return uniqueAliasAst;
}
/**
- * Processes all the tables and subqueries using the given closure.
- *
- * @param from FROM element.
- * @param c Closure each found table and subquery will be passed to. If returns {@code true} the we need to stop.
- * @return {@code true} If we have found.
+ * @param origAlias Original alias.
+ * @return Generated unique alias.
*/
- private static boolean findTablesInFrom(GridSqlElement from, IgnitePredicate<GridSqlElement> c) {
- if (from == null)
- return false;
+ private String nextUniqueTableAlias(String origAlias) {
+ String uniqueAlias = UNIQUE_TABLE_ALIAS_SUFFIX + nextTblAliasId++;
- if (from instanceof GridSqlTable || from instanceof GridSqlSubquery)
- return c.apply(from);
+ if (origAlias != null) // Prepend with the original table alias for better plan readability.
+ uniqueAlias = origAlias + uniqueAlias;
- if (from instanceof GridSqlJoin) {
- // Left and right.
- if (findTablesInFrom(from.child(0), c))
- return true;
+ return uniqueAlias;
+ }
- if (findTablesInFrom(from.child(1), c))
- return true;
- // We don't process ON condition because it is not a joining part of from here.
- return false;
+ /**
+ * @param prnt Parent element.
+ * @param childIdx Child index.
+ * @param prntAlias If the parent is {@link GridSqlAlias}.
+ */
+ private void normalizeFrom(GridSqlAst prnt, int childIdx, boolean prntAlias) {
+ GridSqlElement from = prnt.child(childIdx);
+
+ if (from instanceof GridSqlTable) {
+ GridSqlTable tbl = (GridSqlTable)from;
+
+ String schema = tbl.schema();
+
+ boolean addSchema = tbls == null;
+
+ if (tbls != null)
+ addSchema = tbls.add(tbl.dataTable().identifier());
+
+ if (addSchema && schema != null && schemas != null)
+ schemas.add(schema);
+
+ // In case of alias parent we need to replace the alias itself.
+ if (!prntAlias)
+ generateUniqueAlias(prnt, childIdx);
}
- else if (from instanceof GridSqlAlias)
- return findTablesInFrom(from.child(), c);
- else if (from instanceof GridSqlFunction)
- return false;
+ else if (from instanceof GridSqlAlias) {
+ // Replace current alias with generated unique alias.
+ normalizeFrom(from, 0, true);
+ generateUniqueAlias(prnt, childIdx);
+ }
+ else if (from instanceof GridSqlSubquery) {
+ // We do not need to wrap simple functional subqueries into filtering function,
+ // because we can not have any other tables than Ignite (which are already filtered)
+ // and functions we have to filter explicitly as well.
+ normalizeQuery(((GridSqlSubquery)from).subquery());
+
+ if (!prntAlias) // H2 generates aliases for subqueries in FROM clause.
+ throw new IllegalStateException("No alias for subquery: " + from.getSQL());
+ }
+ else if (from instanceof GridSqlJoin) {
+ // Left and right.
+ normalizeFrom(from, 0, false);
+ normalizeFrom(from, 1, false);
+
+ // Join condition (after ON keyword).
+ normalizeExpression(from, 2);
+ }
+ else if (from instanceof GridSqlFunction) {
+ // TODO generate filtering function around the given function
+ // TODO SYSTEM_RANGE is a special case, it can not be wrapped
- throw new IllegalStateException(from.getClass().getName() + " : " + from.getSQL());
+ // In case of alias parent we need to replace the alias itself.
+ if (!prntAlias)
+ generateUniqueAlias(prnt, childIdx);
+ }
+ else
+ throw new IllegalStateException(from.getClass().getName() + " : " + from.getSQL());
}
/**
- * Searches schema names and tables in subqueries in SELECT and WHERE clauses.
- *
- * @param el Element.
- * @param schemas Schema names.
- * @param tbls Tables.
+ * @param prnt Parent element.
+ * @param childIdx Child index.
*/
- private static void collectAllTablesInSubqueries(GridSqlElement el, Set<String> schemas, Set<String> tbls) {
- if (el == null)
- return;
+ private void normalizeExpression(GridSqlAst prnt, int childIdx) {
+ GridSqlAst el = prnt.child(childIdx);
+
+ if (el instanceof GridSqlAlias ||
+ el instanceof GridSqlOperation ||
+ el instanceof GridSqlFunction ||
+ el instanceof GridSqlArray) {
+ for (int i = 0; i < el.size(); i++)
+ normalizeExpression(el, i);
+ }
+ else if (el instanceof GridSqlSubquery)
+ normalizeQuery(((GridSqlSubquery)el).subquery());
+ else if (el instanceof GridSqlColumn) {
+ GridSqlColumn col = (GridSqlColumn)el;
+ GridSqlAst tbl = GridSqlAlias.unwrap(col.expressionInFrom());
+
+ // Change table alias part of the column to the generated unique table alias.
+ GridSqlAlias uniqueAlias = uniqueFromAliases.get(tbl);
- el = GridSqlAlias.unwrap(el);
+ // Unique aliases must be generated for all the table filters already.
+ assert uniqueAlias != null: childIdx + "\n" + prnt.getSQL();
- if (el instanceof GridSqlOperation || el instanceof GridSqlFunction) {
- for (GridSqlElement child : el)
- collectAllTablesInSubqueries(child, schemas, tbls);
+ col.tableAlias(uniqueAlias.alias());
+ col.expressionInFrom(uniqueAlias);
}
- else if (el instanceof GridSqlSubquery)
- collectAllTables(((GridSqlSubquery)el).select(), schemas, tbls);
+ else if (el instanceof GridSqlParameter ||
+ el instanceof GridSqlPlaceholder ||
+ el instanceof GridSqlConst) {
+ // No-op for simple expressions.
+ }
+ else
+ throw new IllegalStateException(el + ": " + el.getClass());
}
/**
@@ -586,27 +1562,31 @@ public class GridSqlQuerySplitter {
}
/**
- * @param qry Select.
+ * @param select Select.
* @param params Parameters.
* @param target Extracted parameters.
* @param paramIdxs Parameter indexes.
* @return Extracted parameters list.
*/
- private static List<Object> findParams(GridSqlSelect qry, Object[] params, ArrayList<Object> target,
- IntArray paramIdxs) {
+ private static List<Object> findParams(
+ GridSqlSelect select,
+ Object[] params,
+ ArrayList<Object> target,
+ IntArray paramIdxs
+ ) {
if (params.length == 0)
return target;
- for (GridSqlElement el : qry.columns(false))
+ for (GridSqlAst el : select.columns(false))
findParams(el, params, target, paramIdxs);
- findParams(qry.from(), params, target, paramIdxs);
- findParams(qry.where(), params, target, paramIdxs);
+ findParams(select.from(), params, target, paramIdxs);
+ findParams(select.where(), params, target, paramIdxs);
// Don't search in GROUP BY and HAVING since they expected to be in select list.
- findParams(qry.limit(), params, target, paramIdxs);
- findParams(qry.offset(), params, target, paramIdxs);
+ findParams(select.limit(), params, target, paramIdxs);
+ findParams(select.offset(), params, target, paramIdxs);
return target;
}
@@ -617,7 +1597,7 @@ public class GridSqlQuerySplitter {
* @param target Extracted parameters.
* @param paramIdxs Parameter indexes.
*/
- private static void findParams(@Nullable GridSqlElement el, Object[] params, ArrayList<Object> target,
+ private static void findParams(@Nullable GridSqlAst el, Object[] params, ArrayList<Object> target,
IntArray paramIdxs) {
if (el == null)
return;
@@ -644,10 +1624,11 @@ public class GridSqlQuerySplitter {
paramIdxs.add(idx);
}
else if (el instanceof GridSqlSubquery)
- findParams(((GridSqlSubquery)el).select(), params, target, paramIdxs);
- else
- for (GridSqlElement child : el)
- findParams(child, params, target, paramIdxs);
+ findParams(((GridSqlSubquery)el).subquery(), params, target, paramIdxs);
+ else {
+ for (int i = 0; i < el.size(); i++)
+ findParams(el.child(i), params, target, paramIdxs);
+ }
}
/**
@@ -655,15 +1636,21 @@ public class GridSqlQuerySplitter {
* @param rdcSelect Selects for reduce query.
* @param colNames Set of unique top level column names.
* @param idx Index.
- * @param collocated If it is a collocated query.
+ * @param collocatedGrpBy If it is a collocated GROUP BY query.
* @param isHaving If it is a HAVING expression.
* @param hasDistinctAggregate If query has distinct aggregate expression.
* @return {@code true} If aggregate was found.
*/
- private static boolean splitSelectExpression(List<GridSqlElement> mapSelect, List<GridSqlElement> rdcSelect,
- Set<String> colNames, final int idx, boolean collocated, boolean isHaving, boolean hasDistinctAggregate) {
- GridSqlElement el = mapSelect.get(idx);
-
+ private boolean splitSelectExpression(
+ List<GridSqlAst> mapSelect,
+ List<GridSqlAst> rdcSelect,
+ Set<String> colNames,
+ final int idx,
+ boolean collocatedGrpBy,
+ boolean isHaving,
+ boolean hasDistinctAggregate
+ ) {
+ GridSqlAst el = mapSelect.get(idx);
GridSqlAlias alias = null;
boolean aggregateFound = false;
@@ -673,7 +1660,7 @@ public class GridSqlQuerySplitter {
el = alias.child();
}
- if (!collocated && hasAggregates(el)) {
+ if (!collocatedGrpBy && hasAggregates(el)) {
aggregateFound = true;
if (alias == null)
@@ -722,12 +1709,12 @@ public class GridSqlQuerySplitter {
* @param el Expression.
* @return {@code true} If expression contains aggregates.
*/
- private static boolean hasAggregates(GridSqlElement el) {
+ private static boolean hasAggregates(GridSqlAst el) {
if (el instanceof GridSqlAggregateFunction)
return true;
- for (GridSqlElement child : el) {
- if (hasAggregates(child))
+ for (int i = 0; i < el.size(); i++) {
+ if (hasAggregates(el.child(i)))
return true;
}
@@ -741,15 +1728,15 @@ public class GridSqlQuerySplitter {
* @param el Expression.
* @return {@code true} If expression contains distinct aggregates.
*/
- private static boolean hasDistinctAggregates(GridSqlElement el) {
+ private static boolean hasDistinctAggregates(GridSqlAst el) {
if (el instanceof GridSqlAggregateFunction) {
GridSqlFunctionType type = ((GridSqlAggregateFunction)el).type();
return ((GridSqlAggregateFunction)el).distinct() && type != MIN && type != MAX;
}
- for (GridSqlElement child : el) {
- if (hasDistinctAggregates(child))
+ for (int i = 0; i < el.size(); i++) {
+ if (hasDistinctAggregates(el.child(i)))
return true;
}
@@ -765,14 +1752,15 @@ public class GridSqlQuerySplitter {
* @param first If the first aggregate is already found in this expression.
* @return {@code true} If the first aggregate is already found.
*/
- private static boolean splitAggregates(
- final GridSqlElement parentExpr,
+ private boolean splitAggregates(
+ final GridSqlAst parentExpr,
final int childIdx,
- final List<GridSqlElement> mapSelect,
+ final List<GridSqlAst> mapSelect,
final int exprIdx,
boolean hasDistinctAggregate,
- boolean first) {
- GridSqlElement el = parentExpr.child(childIdx);
+ boolean first
+ ) {
+ GridSqlAst el = parentExpr.child(childIdx);
if (el instanceof GridSqlAggregateFunction) {
splitAggregate(parentExpr, childIdx, mapSelect, exprIdx, hasDistinctAggregate, first);
@@ -796,10 +1784,10 @@ public class GridSqlQuerySplitter {
* @param hasDistinctAggregate If query has distinct aggregate expression.
* @param first If this is the first aggregate found in this expression.
*/
- private static void splitAggregate(
- GridSqlElement parentExpr,
+ private void splitAggregate(
+ GridSqlAst parentExpr,
int aggIdx,
- List<GridSqlElement> mapSelect,
+ List<GridSqlAst> mapSelect,
int exprIdx,
boolean hasDistinctAggregate,
boolean first
@@ -925,7 +1913,7 @@ public class GridSqlQuerySplitter {
* @return Column.
*/
private static GridSqlColumn column(String name) {
- return new GridSqlColumn(null, null, name, name);
+ return new GridSqlColumn(null, null, null, null, name);
}
/**
@@ -933,7 +1921,7 @@ public class GridSqlQuerySplitter {
* @param child Child.
* @return Alias.
*/
- private static GridSqlAlias alias(String alias, GridSqlElement child) {
+ private static GridSqlAlias alias(String alias, GridSqlAst child) {
GridSqlAlias res = new GridSqlAlias(alias, child);
res.resultType(child.resultType());
@@ -947,7 +1935,7 @@ public class GridSqlQuerySplitter {
* @param right Right expression.
* @return Binary operator.
*/
- private static GridSqlOperation op(GridSqlOperationType type, GridSqlElement left, GridSqlElement right) {
+ private static GridSqlOperation op(GridSqlOperationType type, GridSqlAst left, GridSqlAst right) {
return new GridSqlOperation(type, left, right);
}
@@ -958,4 +1946,95 @@ public class GridSqlQuerySplitter {
private static GridSqlFunction function(GridSqlFunctionType type) {
return new GridSqlFunction(type);
}
+
+ /**
+ * Simplified tree-like model for a query.
+ * - SELECT : All the children are list of joined query models in the FROM clause.
+ * - UNION : All the children are united left and right query models.
+ * - TABLE and FUNCTION : Never have child models.
+ */
+ private static final class QueryModel extends ArrayList<QueryModel> {
+ /** */
+ final Type type;
+
+ /** */
+ GridSqlAlias uniqueAlias;
+
+ /** */
+ GridSqlAst prnt;
+
+ /** */
+ int childIdx;
+
+ /** If it is a SELECT and we need to split it. Makes sense only for type SELECT. */
+ boolean needSplit;
+
+ /** If we have a child SELECT that we should split. */
+ boolean needSplitChild;
+
+ /** If this is UNION ALL. Makes sense only for type UNION.*/
+ boolean unionAll = true;
+
+ /**
+ * @param type Type.
+ * @param prnt Parent element.
+ * @param childIdx Child index.
+ * @param uniqueAlias Unique parent alias of the current element.
+ * May be {@code null} for selects inside of unions or top level queries.
+ */
+ QueryModel(Type type, GridSqlAst prnt, int childIdx, GridSqlAlias uniqueAlias) {
+ this.type = type;
+ this.prnt = prnt;
+ this.childIdx = childIdx;
+ this.uniqueAlias = uniqueAlias;
+ }
+
+ /**
+ * @return The actual AST element for this model.
+ */
+ private <X extends GridSqlAst> X ast() {
+ return prnt.child(childIdx);
+ }
+
+ /**
+ * @return {@code true} If this is a SELECT or UNION query model.
+ */
+ private boolean isQuery() {
+ return type == Type.SELECT || type == Type.UNION;
+ }
+ }
+
+ /**
+ * Allowed types for {@link QueryModel}.
+ */
+ private enum Type {
+ SELECT, UNION, TABLE, FUNCTION
+ }
+
+ /**
+ * Condition in AND.
+ */
+ private static class AndCondition {
+ /** */
+ GridSqlAst prnt;
+
+ /** */
+ int childIdx;
+
+ /**
+ * @param prnt Parent element.
+ * @param childIdx Child index.
+ */
+ AndCondition(GridSqlAst prnt, int childIdx) {
+ this.prnt = prnt;
+ this.childIdx = childIdx;
+ }
+
+ /**
+ * @return The actual AST element for this expression.
+ */
+ private <X extends GridSqlAst> X ast() {
+ return prnt.child(childIdx);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java
index f49a714..bfa0089 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSelect.java
@@ -27,23 +27,90 @@ import org.h2.util.StringUtils;
*/
public class GridSqlSelect extends GridSqlQuery {
/** */
- private List<GridSqlElement> cols = new ArrayList<>();
+ public static final int FROM_CHILD = 2;
+
+ /** */
+ public static final int WHERE_CHILD = 3;
+
+ /** */
+ private static final int COLS_CHILD = 4;
+
+ /** */
+ private List<GridSqlAst> cols = new ArrayList<>();
/** */
private int visibleCols;
/** */
+ private boolean distinct;
+
+ /** */
private int[] grpCols;
/** */
- private GridSqlElement from;
+ private GridSqlAst from;
/** */
- private GridSqlElement where;
+ private GridSqlAst where;
/** */
private int havingCol = -1;
+ /**
+ * @param colIdx Column index as for {@link #column(int)}.
+ * @return Child index for {@link #child(int)}.
+ */
+ public static int childIndexForColumn(int colIdx) {
+ return colIdx + COLS_CHILD;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int size() {
+ return 4 + cols.size(); // + FROM + WHERE + OFFSET + LIMIT
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public <E extends GridSqlAst> E child(int childIdx) {
+ if (childIdx < FROM_CHILD)
+ return super.child(childIdx);
+
+ switch (childIdx) {
+ case FROM_CHILD:
+ return maskNull(from, GridSqlPlaceholder.EMPTY);
+
+ case WHERE_CHILD:
+ return maskNull(where, GridSqlConst.TRUE);
+
+ default:
+ return (E)cols.get(childIdx - COLS_CHILD);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public <E extends GridSqlAst> void child(int childIdx, E child) {
+ if (childIdx < FROM_CHILD) {
+ super.child(childIdx, child);
+
+ return;
+ }
+
+ switch (childIdx) {
+ case FROM_CHILD:
+ from = child;
+
+ break;
+
+ case WHERE_CHILD:
+ where = child;
+
+ break;
+
+ default:
+ cols.set(childIdx - COLS_CHILD, child);
+ }
+ }
+
/** {@inheritDoc} */
@Override public int visibleColumns() {
return visibleCols;
@@ -57,7 +124,7 @@ public class GridSqlSelect extends GridSqlQuery {
}
/** {@inheritDoc} */
- @Override protected GridSqlElement column(int col) {
+ @Override protected GridSqlAst column(int col) {
return cols.get(col);
}
@@ -68,7 +135,7 @@ public class GridSqlSelect extends GridSqlQuery {
if (distinct)
buff.append(" DISTINCT");
- for (GridSqlElement expression : columns(true)) {
+ for (GridSqlAst expression : columns(true)) {
buff.appendExceptFirst(",");
buff.append('\n');
buff.append(expression.getSQL());
@@ -107,18 +174,18 @@ public class GridSqlSelect extends GridSqlQuery {
* @return {@code True} if this simple SQL query like 'SELECT A, B, C from SOME_TABLE' without any conditions
* and expressions.
*/
- public boolean simpleQuery() {
+ @Override public boolean simpleQuery() {
boolean simple = !distinct &&
from instanceof GridSqlTable &&
where == null &&
grpCols == null &&
havingCol < 0 &&
sort.isEmpty() &&
- limit == null &&
- offset == null;
+ limit() == null &&
+ offset() == null;
if (simple) {
- for (GridSqlElement expression : columns(true)) {
+ for (GridSqlAst expression : columns(true)) {
if (expression instanceof GridSqlAlias)
expression = expression.child();
@@ -134,7 +201,7 @@ public class GridSqlSelect extends GridSqlQuery {
* @param buff Statement builder.
* @param exp Alias expression.
*/
- private static void addAlias(StatementBuilder buff, GridSqlElement exp) {
+ private static void addAlias(StatementBuilder buff, GridSqlAst exp) {
exp = GridSqlAlias.unwrap(exp);
buff.append(StringUtils.unEnclose(exp.getSQL()));
@@ -144,7 +211,7 @@ public class GridSqlSelect extends GridSqlQuery {
* @param visibleOnly If only visible expressions needed.
* @return Select clause expressions.
*/
- public List<GridSqlElement> columns(boolean visibleOnly) {
+ public List<GridSqlAst> columns(boolean visibleOnly) {
assert visibleCols <= cols.size();
return visibleOnly && visibleCols != cols.size() ?
@@ -167,7 +234,7 @@ public class GridSqlSelect extends GridSqlQuery {
* @param visible Expression is visible in select phrase.
* @return {@code this}.
*/
- public GridSqlSelect addColumn(GridSqlElement expression, boolean visible) {
+ public GridSqlSelect addColumn(GridSqlAst expression, boolean visible) {
if (expression == null)
throw new NullPointerException();
@@ -188,7 +255,7 @@ public class GridSqlSelect extends GridSqlQuery {
* @param expression Expression.
* @return {@code this}.
*/
- public GridSqlSelect setColumn(int colIdx, GridSqlElement expression) {
+ public GridSqlSelect setColumn(int colIdx, GridSqlAst expression) {
if (expression == null)
throw new NullPointerException();
@@ -217,7 +284,7 @@ public class GridSqlSelect extends GridSqlQuery {
/**
* @return Tables.
*/
- public GridSqlElement from() {
+ public GridSqlAst from() {
return from;
}
@@ -225,16 +292,30 @@ public class GridSqlSelect extends GridSqlQuery {
* @param from From element.
* @return {@code this}.
*/
- public GridSqlSelect from(GridSqlElement from) {
+ public GridSqlSelect from(GridSqlAst from) {
this.from = from;
return this;
}
/**
+ * @return Distinct.
+ */
+ public boolean distinct() {
+ return distinct;
+ }
+
+ /**
+ * @param distinct New distinct.
+ */
+ public void distinct(boolean distinct) {
+ this.distinct = distinct;
+ }
+
+ /**
* @return Where.
*/
- public GridSqlElement where() {
+ public GridSqlAst where() {
return where;
}
@@ -242,7 +323,7 @@ public class GridSqlSelect extends GridSqlQuery {
* @param where New where.
* @return {@code this}.
*/
- public GridSqlSelect where(GridSqlElement where) {
+ public GridSqlSelect where(GridSqlAst where) {
this.where = where;
return this;
@@ -252,11 +333,11 @@ public class GridSqlSelect extends GridSqlQuery {
* @param cond Adds new WHERE condition using AND operator.
* @return {@code this}.
*/
- public GridSqlSelect whereAnd(GridSqlElement cond) {
+ public GridSqlSelect whereAnd(GridSqlAst cond) {
if (cond == null)
throw new NullPointerException();
- GridSqlElement old = where();
+ GridSqlAst old = where();
where(old == null ? cond : new GridSqlOperation(GridSqlOperationType.AND, old, cond));
@@ -266,7 +347,7 @@ public class GridSqlSelect extends GridSqlQuery {
/**
* @return Having.
*/
- public GridSqlElement having() {
+ public GridSqlAst having() {
return havingCol >= 0 ? column(havingCol) : null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
index 6eda0d7..21cf596 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
@@ -22,7 +22,7 @@ package org.apache.ignite.internal.processors.query.h2.sql;
*/
public abstract class GridSqlStatement {
/** */
- protected GridSqlElement limit;
+ protected GridSqlAst limit;
/** */
private boolean explain;
@@ -51,14 +51,14 @@ public abstract class GridSqlStatement {
/**
* @param limit Limit.
*/
- public void limit(GridSqlElement limit) {
+ public void limit(GridSqlAst limit) {
this.limit = limit;
}
/**
* @return Limit.
*/
- public GridSqlElement limit() {
+ public GridSqlAst limit() {
return limit;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
index 7eb9eab..887e427 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
@@ -17,40 +17,31 @@
package org.apache.ignite.internal.processors.query.h2.sql;
-import java.util.Collections;
+import java.util.ArrayList;
+
/**
- * Subquery.
+ * Subquery expression.
*/
public class GridSqlSubquery extends GridSqlElement {
- /** */
- private GridSqlQuery select;
-
/**
- * @param select Select.
+ * @param subQry Subquery.
*/
- public GridSqlSubquery(GridSqlQuery select) {
- super(Collections.<GridSqlElement>emptyList());
+ public GridSqlSubquery(GridSqlQuery subQry) {
+ super(new ArrayList<GridSqlAst>(1));
- this.select = select;
+ addChild(subQry);
}
/** {@inheritDoc} */
@Override public String getSQL() {
- return "(" + select.getSQL() + ")";
- }
-
- /**
- * @return Select.
- */
- public GridSqlQuery select() {
- return select;
+ return "(" + subquery().getSQL() + ")";
}
/**
- * @param select New select.
+ * @return Subquery AST.
*/
- public void select(GridSqlQuery select) {
- this.select = select;
+ public GridSqlQuery subquery() {
+ return child(0);
}
}
\ No newline at end of file
[45/50] [abbrv] ignite git commit: IGNITE-4752 Hadoop examples fail
for hadoop accelerator
Posted by av...@apache.org.
IGNITE-4752 Hadoop examples fail for hadoop accelerator
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4dd3d23b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4dd3d23b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4dd3d23b
Branch: refs/heads/master
Commit: 4dd3d23b96e8ff348f603e63d99905fc5dca6c4a
Parents: 5729177
Author: Evgenii Zhuravlev <ez...@gridgain.com>
Authored: Mon Feb 27 13:37:18 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Mon Feb 27 14:08:33 2017 +0300
----------------------------------------------------------------------
assembly/dependencies-hadoop.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4dd3d23b/assembly/dependencies-hadoop.xml
----------------------------------------------------------------------
diff --git a/assembly/dependencies-hadoop.xml b/assembly/dependencies-hadoop.xml
index 38646ba..076f09c 100644
--- a/assembly/dependencies-hadoop.xml
+++ b/assembly/dependencies-hadoop.xml
@@ -55,6 +55,9 @@
<fileSet>
<directory>target/libs</directory>
<outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>hadoop*.jar</exclude>
+ </excludes>
</fileSet>
<fileSet>
<directory>target</directory>
[08/50] [abbrv] ignite git commit: IGNITE-3710 Upgrade ignite-spark
module to Spark 2.0 (cherry picked from commit 8613c16)
Posted by av...@apache.org.
IGNITE-3710 Upgrade ignite-spark module to Spark 2.0
(cherry picked from commit 8613c16)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/caf7b225
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/caf7b225
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/caf7b225
Branch: refs/heads/master
Commit: caf7b225c6ae55a530961a3b79ebde2368b6a24d
Parents: 5b94a7d
Author: Evgenii Zhuravlev <ez...@gridgain.com>
Authored: Mon Feb 20 19:24:42 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Mon Feb 20 19:26:17 2017 +0300
----------------------------------------------------------------------
.../examples/java8/spark/SharedRDDExample.java | 4 +-
modules/spark-2.10/pom.xml | 54 ++++++
modules/spark/pom.xml | 183 ++++++++++++++++++-
.../org/apache/ignite/spark/IgniteContext.scala | 22 ++-
.../spark/JavaEmbeddedIgniteRDDSelfTest.java | 10 +-
.../spark/JavaStandaloneIgniteRDDSelfTest.java | 22 +--
parent/pom.xml | 3 +-
7 files changed, 270 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/caf7b225/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java b/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java
index 392180d..5f74a94 100644
--- a/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java
+++ b/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java
@@ -26,7 +26,7 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Dataset;
import scala.Tuple2;
import java.util.List;
@@ -99,7 +99,7 @@ public class SharedRDDExample {
System.out.println(">>> Executing SQL query over Ignite Shared RDD...");
// Execute SQL query over the Ignite RDD.
- DataFrame df = sharedRDD.sql("select _val from Integer where _key < 9");
+ Dataset df = sharedRDD.sql("select _val from Integer where _key < 9");
// Show the result of the execution.
df.show();
http://git-wip-us.apache.org/repos/asf/ignite/blob/caf7b225/modules/spark-2.10/pom.xml
----------------------------------------------------------------------
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index ed42227..58e2860 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -63,10 +63,64 @@
<dependency>
<groupId>org.apache.spark</groupId>
+ <artifactId>spark-unsafe_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-network-shuffle_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-tags_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-catalyst_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.json4s</groupId>
+ <artifactId>json4s-core_2.11</artifactId>
+ <version>3.5.0</version>
+ </dependency>
+
<!-- Test dependencies -->
<dependency>
http://git-wip-us.apache.org/repos/asf/ignite/blob/caf7b225/modules/spark/pom.xml
----------------------------------------------------------------------
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index d8cb894..140f637 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -52,7 +52,7 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>2.11.7</version>
+ <version>2.11.8</version>
</dependency>
<dependency>
@@ -63,11 +63,53 @@
<dependency>
<groupId>org.apache.spark</groupId>
+ <artifactId>spark-catalyst_2.11</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-network-common_2.11</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-network-shuffle_2.11</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.json4s</groupId>
+ <artifactId>json4s-core_2.11</artifactId>
+ <version>3.5.0</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-indexing</artifactId>
<version>${project.version}</version>
@@ -89,7 +131,7 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
- <version>2.2.4</version>
+ <version>2.2.6</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -98,6 +140,143 @@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-unsafe_2.11</artifactId>
+ <version>${spark.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-launcher_2.11</artifactId>
+ <version>${spark.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-tags_2.11</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-unsafe_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.5</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.0.29.Final</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.esotericsoftware.kryo</groupId>
+ <artifactId>kryo</artifactId>
+ <version>2.20</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>chill_2.11</artifactId>
+ <version>0.8.1</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ <version>3.1.2</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-servlet-api</artifactId>
+ <version>8.0.23</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet-core</artifactId>
+ <version>2.25</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson2.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-scala_2.11</artifactId>
+ <version>${jackson2.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.xbean</groupId>
+ <artifactId>xbean-asm5-shaded</artifactId>
+ <version>4.5</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ <version>1.3.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop</artifactId>
+ <version>1.9.0</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>janino</artifactId>
+ <version>3.0.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.janino</groupId>
+ <artifactId>commons-compiler</artifactId>
+ <version>3.0.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/ignite/blob/caf7b225/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
----------------------------------------------------------------------
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
index 04139d1..842c459 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteContext.scala
@@ -22,7 +22,8 @@ import org.apache.ignite.configuration.{CacheConfiguration, IgniteConfiguration}
import org.apache.ignite.internal.IgnitionEx
import org.apache.ignite.internal.util.IgniteUtils
import org.apache.spark.sql.SQLContext
-import org.apache.spark.{Logging, SparkContext}
+import org.apache.spark.SparkContext
+import org.apache.log4j.Logger
/**
* Ignite context.
@@ -34,7 +35,7 @@ class IgniteContext(
@transient val sparkContext: SparkContext,
cfgF: () \u21d2 IgniteConfiguration,
standalone: Boolean = true
- ) extends Serializable with Logging {
+ ) extends Serializable {
private val cfgClo = new Once(cfgF)
private val igniteHome = IgniteUtils.getIgniteHome
@@ -47,7 +48,7 @@ class IgniteContext(
if (workers <= 0)
throw new IllegalStateException("No Spark executors found to start Ignite nodes.")
- logInfo("Will start Ignite nodes on " + workers + " workers")
+ Logging.log.info("Will start Ignite nodes on " + workers + " workers")
// Start ignite server node on each worker in server mode.
sparkContext.parallelize(1 to workers, workers).foreachPartition(it \u21d2 ignite())
@@ -126,7 +127,7 @@ class IgniteContext(
val home = IgniteUtils.getIgniteHome
if (home == null && igniteHome != null) {
- logInfo("Setting IGNITE_HOME from driver not as it is not available on this worker: " + igniteHome)
+ Logging.log.info("Setting IGNITE_HOME from driver not as it is not available on this worker: " + igniteHome)
IgniteUtils.nullifyHomeDirectory()
@@ -143,7 +144,7 @@ class IgniteContext(
}
catch {
case e: IgniteException \u21d2
- logError("Failed to start Ignite.", e)
+ Logging.log.error("Failed to start Ignite.", e)
throw e
}
@@ -161,7 +162,7 @@ class IgniteContext(
sparkContext.getExecutorStorageStatus.length)
if (workers > 0) {
- logInfo("Will stop Ignite nodes on " + workers + " workers")
+ Logging.log.info("Will stop Ignite nodes on " + workers + " workers")
// Start ignite server node on each worker in server mode.
sparkContext.parallelize(1 to workers, workers).foreachPartition(it \u21d2 doClose())
@@ -200,3 +201,12 @@ private class Once(clo: () \u21d2 IgniteConfiguration) extends Serializable {
res
}
}
+
+/**
+ * Spark uses log4j by default. Using this logger in IgniteContext as well.
+ *
+ * This object is used to avoid problems with log4j serialization.
+ */
+object Logging extends Serializable {
+ @transient lazy val log = Logger.getLogger(classOf[IgniteContext])
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/caf7b225/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java
index 0c4d556..53aff75 100644
--- a/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java
+++ b/modules/spark/src/test/java/org/apache/ignite/spark/JavaEmbeddedIgniteRDDSelfTest.java
@@ -35,7 +35,7 @@ import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.Column;
-import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import scala.Tuple2;
@@ -237,12 +237,12 @@ public class JavaEmbeddedIgniteRDDSelfTest extends GridCommonAbstractTest {
cache.savePairs(sc.parallelize(F.range(0, 1001), GRID_CNT).mapToPair(INT_TO_ENTITY_F), true);
- DataFrame df =
+ Dataset<Row> df =
cache.sql("select id, name, salary from Entity where name = ? and salary = ?", "name50", 5000);
df.printSchema();
- Row[] res = df.collect();
+ Row[] res = (Row[])df.collect();
assertEquals("Invalid result length", 1, res.length);
assertEquals("Invalid result", 50, res[0].get(0));
@@ -251,11 +251,11 @@ public class JavaEmbeddedIgniteRDDSelfTest extends GridCommonAbstractTest {
Column exp = new Column("NAME").equalTo("name50").and(new Column("SALARY").equalTo(5000));
- DataFrame df0 = cache.sql("select id, name, salary from Entity").where(exp);
+ Dataset<Row> df0 = cache.sql("select id, name, salary from Entity").where(exp);
df.printSchema();
- Row[] res0 = df0.collect();
+ Row[] res0 = (Row[])df0.collect();
assertEquals("Invalid result length", 1, res0.length);
assertEquals("Invalid result", 50, res0[0].get(0));
http://git-wip-us.apache.org/repos/asf/ignite/blob/caf7b225/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java b/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
index e9d97a4..1075f96 100644
--- a/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
+++ b/modules/spark/src/test/java/org/apache/ignite/spark/JavaStandaloneIgniteRDDSelfTest.java
@@ -35,7 +35,7 @@ import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.Column;
-import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import scala.Tuple2;
@@ -216,12 +216,12 @@ public class JavaStandaloneIgniteRDDSelfTest extends GridCommonAbstractTest {
cache.savePairs(sc.parallelize(F.range(0, 1001), 2).mapToPair(INT_TO_ENTITY_F));
- DataFrame df =
+ Dataset<Row> df =
cache.sql("select id, name, salary from Entity where name = ? and salary = ?", "name50", 5000);
df.printSchema();
- Row[] res = df.collect();
+ Row[] res = (Row[])df.collect();
assertEquals("Invalid result length", 1, res.length);
assertEquals("Invalid result", 50, res[0].get(0));
@@ -230,11 +230,11 @@ public class JavaStandaloneIgniteRDDSelfTest extends GridCommonAbstractTest {
Column exp = new Column("NAME").equalTo("name50").and(new Column("SALARY").equalTo(5000));
- DataFrame df0 = cache.sql("select id, name, salary from Entity").where(exp);
+ Dataset<Row> df0 = cache.sql("select id, name, salary from Entity").where(exp);
df.printSchema();
- Row[] res0 = df0.collect();
+ Row[] res0 = (Row[])df0.collect();
assertEquals("Invalid result length", 1, res0.length);
assertEquals("Invalid result", 50, res0[0].get(0));
@@ -269,25 +269,25 @@ public class JavaStandaloneIgniteRDDSelfTest extends GridCommonAbstractTest {
Object val = GridTestUtils.getFieldValue(e, fieldName);
- DataFrame df = cache.sql(
+ Dataset<Row> df = cache.sql(
String.format("select %s from EntityTestAllTypeFields where %s = ?", fieldName, fieldName),
val);
if (val instanceof BigDecimal) {
- Object res = df.collect()[0].get(0);
+ Object res = ((Row[])df.collect())[0].get(0);
assertTrue(String.format("+++ Fail on %s field", fieldName),
((Comparable<BigDecimal>)val).compareTo((BigDecimal)res) == 0);
}
else if (val instanceof java.sql.Date)
assertEquals(String.format("+++ Fail on %s field", fieldName),
- val.toString(), df.collect()[0].get(0).toString());
+ val.toString(), ((Row[])df.collect())[0].get(0).toString());
else if (val.getClass().isArray())
assertTrue(String.format("+++ Fail on %s field", fieldName), 1 <= df.count());
else {
- assertTrue(String.format("+++ Fail on %s field", fieldName), df.collect().length > 0);
- assertTrue(String.format("+++ Fail on %s field", fieldName), df.collect()[0].size() > 0);
- assertEquals(String.format("+++ Fail on %s field", fieldName), val, df.collect()[0].get(0));
+ assertTrue(String.format("+++ Fail on %s field", fieldName), ((Row[])df.collect()).length > 0);
+ assertTrue(String.format("+++ Fail on %s field", fieldName), ((Row[])df.collect())[0].size() > 0);
+ assertEquals(String.format("+++ Fail on %s field", fieldName), val, ((Row[])df.collect())[0].get(0));
}
info(String.format("+++ Query on the filed: %s : %s passed", fieldName, f.getType().getSimpleName()));
http://git-wip-us.apache.org/repos/asf/ignite/blob/caf7b225/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 3514435..a4972d1 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -36,7 +36,7 @@
<properties>
<ignite.edition>fabric</ignite.edition>
<hadoop.version>2.4.1</hadoop.version>
- <spark.version>1.5.2</spark.version>
+ <spark.version>2.1.0</spark.version>
<spring.version>4.1.0.RELEASE</spring.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.build.timestamp.format>MMMM d yyyy</maven.build.timestamp.format>
@@ -99,7 +99,6 @@
<scala211.library.version>2.11.7</scala211.library.version>
<slf4j.version>1.7.7</slf4j.version>
<slf4j16.version>1.6.4</slf4j16.version>
- <spark.version>1.5.2</spark.version>
<spring.version>4.1.0.RELEASE</spring.version>
<spring41.osgi.feature.version>4.1.7.RELEASE_1</spring41.osgi.feature.version>
<tomcat.version>8.0.23</tomcat.version>
[05/50] [abbrv] ignite git commit: IGNITE-4717 Fixed hangs in
VisorCacheClearTask.
Posted by av...@apache.org.
IGNITE-4717 Fixed hangs in VisorCacheClearTask.
(cherry picked from commit 76f3060)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/95dd7482
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/95dd7482
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/95dd7482
Branch: refs/heads/master
Commit: 95dd74829339c05b3f092829a3f84fadd84f67f2
Parents: 0130b09
Author: Andrey Novikov <an...@gridgain.com>
Authored: Mon Feb 20 18:23:33 2017 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Mon Feb 20 18:29:58 2017 +0700
----------------------------------------------------------------------
.../visor/cache/VisorCacheClearTask.java | 88 +++++---------------
.../visor/compute/VisorGatewayTask.java | 30 ++++++-
2 files changed, 49 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/95dd7482/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
index 1f1a6fb..0c8476f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheClearTask.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.visor.cache;
import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCompute;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.compute.ComputeJobContext;
import org.apache.ignite.internal.processors.task.GridInternal;
@@ -26,7 +25,6 @@ import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.visor.VisorJob;
import org.apache.ignite.internal.visor.VisorOneNodeTask;
import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.resources.JobContextResource;
@@ -90,17 +88,11 @@ public class VisorCacheClearTask extends VisorOneNodeTask<String, IgniteBiTuple<
}
/**
- * @param subJob Sub job to execute asynchronously.
+ * @param fut Future for asynchronous cache operation.
* @param idx Index.
* @return {@code true} If subJob was not completed and this job should be suspended.
*/
- private boolean callAsync(IgniteCallable<Integer> subJob, int idx) {
- IgniteCompute compute = ignite.compute(ignite.cluster().forCacheNodes(cacheName)).withAsync();
-
- compute.call(subJob);
-
- IgniteFuture<Integer> fut = compute.future();
-
+ private boolean callAsync(IgniteFuture<Integer> fut, int idx) {
futs[idx] = fut;
if (fut.isDone())
@@ -119,16 +111,28 @@ public class VisorCacheClearTask extends VisorOneNodeTask<String, IgniteBiTuple<
futs = new IgniteFuture[3];
if (futs[0] == null || futs[1] == null || futs[2] == null) {
- IgniteCache cache = ignite.cache(cacheName);
+ IgniteCache cache = ignite.cache(cacheName).withAsync();
+
+ if (futs[0] == null) {
+ cache.size(CachePeekMode.PRIMARY);
+
+ if (callAsync(cache.<Integer>future(), 0))
+ return null;
+ }
- if (futs[0] == null && callAsync(new VisorCacheSizeCallable(cache), 0))
- return null;
+ if (futs[1] == null) {
+ cache.clear();
- if (futs[1] == null && callAsync(new VisorCacheClearCallable(cache), 1))
- return null;
+ if (callAsync(cache.<Integer>future(), 1))
+ return null;
+ }
+
+ if (futs[2] == null) {
+ cache.size(CachePeekMode.PRIMARY);
- if (futs[2] == null && callAsync(new VisorCacheSizeCallable(cache), 2))
- return null;
+ if (callAsync(cache.<Integer>future(), 2))
+ return null;
+ }
}
assert futs[0].isDone() && futs[1].isDone() && futs[2].isDone();
@@ -141,54 +145,4 @@ public class VisorCacheClearTask extends VisorOneNodeTask<String, IgniteBiTuple<
return S.toString(VisorCacheClearJob.class, this);
}
}
-
- /**
- * Callable to get cache size.
- */
- @GridInternal
- private static class VisorCacheSizeCallable implements IgniteCallable<Integer> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private final IgniteCache cache;
-
- /**
- * @param cache Cache to take size from.
- */
- private VisorCacheSizeCallable(IgniteCache cache) {
- this.cache = cache;
- }
-
- /** {@inheritDoc} */
- @Override public Integer call() throws Exception {
- return cache.size(CachePeekMode.PRIMARY);
- }
- }
-
- /**
- * Callable to clear cache.
- */
- @GridInternal
- private static class VisorCacheClearCallable implements IgniteCallable<Integer> {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private final IgniteCache cache;
-
- /**
- * @param cache Cache to clear.
- */
- private VisorCacheClearCallable(IgniteCache cache) {
- this.cache = cache;
- }
-
- /** {@inheritDoc} */
- @Override public Integer call() throws Exception {
- cache.clear();
-
- return 0;
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/95dd7482/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index 2539a26..a64ec6d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -29,21 +29,26 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.ignite.IgniteCompute;
import org.apache.ignite.IgniteException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.compute.ComputeJob;
import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobContext;
import org.apache.ignite.compute.ComputeJobResult;
import org.apache.ignite.compute.ComputeJobResultPolicy;
import org.apache.ignite.compute.ComputeTask;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.lang.GridTuple3;
+import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.visor.VisorTaskArgument;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.JobContextResource;
import org.jetbrains.annotations.Nullable;
/**
@@ -101,9 +106,16 @@ public class VisorGatewayTask implements ComputeTask<Object[], Object> {
@IgniteInstanceResource
protected transient IgniteEx ignite;
+ /** Auto-inject job context. */
+ @JobContextResource
+ protected transient ComputeJobContext jobCtx;
+
/** Arguments count. */
private final int argsCnt;
+ /** Future for spawned task. */
+ private transient IgniteFuture fut;
+
/**
* Create job with specified argument.
*
@@ -284,6 +296,9 @@ public class VisorGatewayTask implements ComputeTask<Object[], Object> {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override public Object execute() throws IgniteException {
+ if (fut != null)
+ return fut.get();
+
String nidsArg = argument(0);
String taskName = argument(1);
@@ -355,8 +370,19 @@ public class VisorGatewayTask implements ComputeTask<Object[], Object> {
}
}
- return ignite.compute(ignite.cluster().forNodeIds(nids))
- .execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+ IgniteCompute comp = ignite.compute(ignite.cluster().forNodeIds(nids)).withAsync();
+
+ comp.execute(taskName, new VisorTaskArgument<>(nids, jobArgs, false));
+
+ fut = comp.future();
+
+ fut.listen(new CI1<IgniteFuture<Object>>() {
+ @Override public void apply(IgniteFuture<Object> f) {
+ jobCtx.callcc();
+ }
+ });
+
+ return jobCtx.holdcc();
}
}
}
[22/50] [abbrv] ignite git commit: Minors
Posted by av...@apache.org.
Minors
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2e90a072
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2e90a072
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2e90a072
Branch: refs/heads/master
Commit: 2e90a07218272c5cb2c09111e193a3613d3d51ac
Parents: 56ce223
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 15:05:51 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 15:05:51 2017 +0300
----------------------------------------------------------------------
.../query/h2/opt/GridH2TreeIndex.java | 34 ++++----------------
1 file changed, 6 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2e90a072/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 64ca9ea..07c3e6d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -45,25 +45,13 @@ import org.h2.table.TableFilter;
import org.h2.value.Value;
import org.jetbrains.annotations.Nullable;
+import static org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.KEY_COL;
+
/**
* Base class for snapshotable segmented tree indexes.
*/
@SuppressWarnings("ComparatorNotSerializable")
public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridSearchRowPointer> {
- /** */
- private static Field KEY_FIELD;
-
- /** */
- static {
- try {
- KEY_FIELD = GridH2AbstractKeyValueRow.class.getDeclaredField("key");
- KEY_FIELD.setAccessible(true);
- }
- catch (NoSuchFieldException e) {
- KEY_FIELD = null;
- }
- }
-
/** Index segments. */
private final ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row>[] segments;
@@ -451,22 +439,12 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
assert ctx != null;
- if (row instanceof GridH2AbstractKeyValueRow && KEY_FIELD != null) {
- try {
- Object o = KEY_FIELD.get(row);
-
- if (o instanceof CacheObject)
- key = (CacheObject)o;
- else
- key = ctx.toCacheKeyObject(o);
+ Object o = ctx.toCacheKeyObject(row.getValue(KEY_COL));
- }
- catch (IllegalAccessException e) {
- throw new IllegalStateException(e);
- }
- }
+ if (o instanceof CacheObject)
+ key = (CacheObject)o;
else
- key = ctx.toCacheKeyObject(row.getValue(0));
+ key = ctx.toCacheKeyObject(o);
return segmentForPartition(ctx.affinity().partition(key));
}
[03/50] [abbrv] ignite git commit: IGNITE-4212 Ignite Benchmarking
Simplification and Automation / IGNITE-4478 Fixing documentation and
resolving lack of usability (cherry picked from commit 1265793)
Posted by av...@apache.org.
IGNITE-4212 Ignite Benchmarking Simplification and Automation / IGNITE-4478 Fixing documentation and resolving lack of usability
(cherry picked from commit 1265793)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1a08ef76
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1a08ef76
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1a08ef76
Branch: refs/heads/master
Commit: 1a08ef76bdfb9f4861ad97a2d3c94c6bd98dec82
Parents: 9eb26bd
Author: Author: oleg-ostanin <oo...@gridgain.com>
Authored: Fri Feb 17 19:51:50 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Fri Feb 17 19:53:50 2017 +0300
----------------------------------------------------------------------
modules/yardstick/DEVNOTES-standalone.txt | 5 +-
modules/yardstick/README.txt | 85 +++++++------
.../config/benchmark-atomic-win.properties | 2 +-
.../config/benchmark-atomic.properties | 35 ++++--
.../config/benchmark-bin-identity.properties | 16 +--
.../config/benchmark-cache-load-win.properties | 2 +-
.../config/benchmark-cache-load.properties | 4 +-
.../config/benchmark-client-mode.properties | 68 +++++++----
.../config/benchmark-compute-win.properties | 2 +-
.../config/benchmark-compute.properties | 30 ++++-
.../config/benchmark-failover.properties | 2 +-
.../yardstick/config/benchmark-full.properties | 62 +++++-----
.../config/benchmark-multicast.properties | 107 +++++++++--------
.../config/benchmark-put-indexed-val.properties | 23 +++-
.../benchmark-query-put-separated.properties | 3 +-
.../config/benchmark-query-win.properties | 2 +-
.../yardstick/config/benchmark-query.properties | 33 +++--
.../config/benchmark-remote-sample.properties | 80 +++++++++++++
.../config/benchmark-remote.properties | 119 +++++++++++++++++++
.../config/benchmark-sample.properties | 22 +++-
.../config/benchmark-sql-dml.properties | 36 ++++--
.../yardstick/config/benchmark-store.properties | 3 +-
.../config/benchmark-tx-win.properties | 2 +-
.../yardstick/config/benchmark-tx.properties | 33 +++--
.../yardstick/config/benchmark-win.properties | 2 +-
modules/yardstick/config/benchmark.properties | 76 +++++++-----
.../yardstick/config/ignite-remote-config.xml | 47 ++++++++
.../test-max-int-values-offheap.properties | 3 +-
.../test-max-int-values-onheap.properties | 3 +-
.../config/test-max-int-values-swap.properties | 3 +-
modules/yardstick/pom-standalone.xml | 2 +-
modules/yardstick/pom.xml | 2 +-
pom.xml | 5 +-
33 files changed, 660 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/DEVNOTES-standalone.txt
----------------------------------------------------------------------
diff --git a/modules/yardstick/DEVNOTES-standalone.txt b/modules/yardstick/DEVNOTES-standalone.txt
index d14dde2..50aa7c4 100644
--- a/modules/yardstick/DEVNOTES-standalone.txt
+++ b/modules/yardstick/DEVNOTES-standalone.txt
@@ -11,5 +11,6 @@ Artifacts can be found in `/target/assembly directory.`
Writing Ignite Benchmarks
=========================
-All benchmarks extend `AbstractBenchmark` class. A new benchmark should also extend this abstract class
-and implement `test` method. This is the method which actually tests performance.
\ No newline at end of file
+All benchmarks extend `AbstractBenchmark` class. A new benchmark should also extend
+this abstract class and implement `test` method. This method is the entry point for
+a benchmark execution.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/README.txt
----------------------------------------------------------------------
diff --git a/modules/yardstick/README.txt b/modules/yardstick/README.txt
index dcfe0e6..5903fd4 100644
--- a/modules/yardstick/README.txt
+++ b/modules/yardstick/README.txt
@@ -1,61 +1,73 @@
Yardstick Ignite Benchmarks
===========================
-Apache Ignite benchmarks are written on top of Yardstick Framework (https://github.com/gridgain/yardstick) and allow
-you to measure performance of various Apache Ignite components and modules.
+Apache Ignite benchmarks are written on top of Yardstick Framework (https://github.com/gridgain/yardstick)
+and allow you to measure performance of various Apache Ignite components and modules.
The documentation below describes how to execute and configure already assembled benchmarks. If you need to add new
benchmarks or build existing one then refer to instruction from `DEVNOTES.txt` in source directory.
-Visit Yardstick Repository (https://github.com/gridgain/yardstick) for detailed information in regards resulting graphs
-generation and how the frameworks works.
+Visit Yardstick Repository (https://github.com/gridgain/yardstick) for detailed information in regards resulting
+graphs generation and how the frameworks works.
Running Ignite Benchmarks Locally
-==========================================
+=================================
-The simplest way to start with benchmarking is to use one of the executable scripts available under `benchmarks\bin` directory:
+The simplest way to start with benchmarking is to use one of the executable scripts available under `benchmarks\bin`
+directory:
-./bin/benchmark-run-all.sh config/benchmark-atomic-put.properties
+./bin/benchmark-run-all.sh config/benchmark-sample.properties
-The command above will benchmark the cache put operation for a distributed atomic cache. The results of the benchmark will be added
-to auto-generated `results-{DATE-TIME}` directory.
+The command above will benchmark the cache put operation for a distributed atomic cache. The results of the
+benchmark will be added to an auto-generated `output/results-{DATE-TIME}` directory.
-If `./bin/benchmark-run-all.sh` command is executed as is without any parameters and modifications in configurations files then
-all the available benchmarks will be executed on a local machine using `config/benchmark.properties` configuration.
+If `./bin/benchmark-run-all.sh` command is executed as is without any parameters and modifications in configurations
+files then all the available benchmarks will be executed on a local machine using `config/benchmark.properties` configuration.
-To get more information about available benchmarks and configuration parameters refer to \u201cProvided Benchmarks\u201d and
-\u201cProperties And Command Line Arguments\u201d sections below.
+In case of any issue refer to the logs that are added to an auto-generated
+`output/logs-{DATE-TIME}` directory.
+
+To get more information about available benchmarks and configuration parameters refer
+to \u201cProvided Benchmarks\u201d and \u201cProperties And Command Line Arguments\u201d sections below.
Running Ignite Benchmarks Remotely
-=========================================
+==================================
+
+To benchmark Apache Ignite across several remote hosts the following steps need
+to be done:
-For running Ignite benchmarks on remote hosts you need to upload Ignite-Yardstick to each one of your remote hosts.
+1. Go to `config/ignite-remote-config.xml` and replace
+`<value>127.0.0.1:47500..47509</value>` with actual IPs of all the remote hosts.
+Refer to the documentation section below if you prefer to use other kind of IP finder:
+https://apacheignite.readme.io/docs/cluster-config
-NOTE: The path to the uploaded Ignite-Yardstick should be exactly the same on each host.
+2. Go to `config/benchmark-remote-sample.properties` and replace the `localhost` with
+actual IPs of the remote hosts in the following places:
+SERVERS='localhost,localhost'
+DRIVERS='localhost'
-Then you need to make some changes in config files:
+DRIVERS is the remote hosts where benchmarks execution will be and driven.
-1. You need to comment or delete the
- <property name="localHost" value="127.0.0.1"/>
-line in `config/ignite-localhost-config.xml` file.
+Replace `localhost` occurrences in the same places in
+`config/benchmark-remote.properties` files if you plan to execute a full set of
+benchmarks available.
-2. You need to replace all the `127.0.0.1` addresses in `ignite-localhost-config.xml` file by actual IPs of your remote
-servers. You can add or delete lines with IP addresses if you want to run benchmarks on different number of hosts.
-There must be at least one IP address in the list.
+3. Upload Ignite Yardstick Benchmarks to one of your DRIVERS host in its own working directory.
-3. You need to replace the `localhost` strings by actual IP of your servers in the lines
-SERVERS='localhost,localhost'
-DRIVERS='localhost'
-in `config/benchmark-atomic-put.properties` file.
+4. Log in on the remote host that will be the DRIVER and execute the following command:
-Then use the following command:
+./bin/benchmark-run-all.sh config/benchmark-remote-sample.properties
-./bin/benchmark-run-all.sh config/benchmark-atomic-put.properties
+By default, all the necessary files will be automatically uploaded from the host in which you run the command above to
+every other host to the same path. If you prefer to do it manually set the AUTO_COPY variable in property file to `false`.
+The command above will benchmark the cache put operation for a distributed atomic cache. The results of the benchmark
+will be added to an auto-generated `output/results-{DATE-TIME}` directory.
-It is recommended to create some copies of original config files, edit these copies and then use as a
-parameter for `./bin/benchmark-run-all.sh` script.
+If you want to execute all the available benchmarks across the remote hosts then
+execute the following command on the DRIVER side:
+./bin/benchmark-run-all.sh config/benchmark-remote.properties
Provided Benchmarks
@@ -90,9 +102,9 @@ The following benchmarks are provided:
Properties And Command Line Arguments
=====================================
-Note that this section only describes configuration parameters specific to Ignite benchmarks,
-and not for Yardstick framework. To run Ignite benchmarks and generate graphs, you will need to run them using
-Yardstick framework scripts in `bin` folder.
+Note that this section only describes configuration parameters specific to Ignite benchmarks, and not for Yardstick
+framework. To run Ignite benchmarks and generate graphs, you will need to run them using Yardstick framework scripts in
+`bin` folder.
Refer to Yardstick Documentation (https://github.com/gridgain/yardstick) for common Yardstick properties
and command line arguments for running Yardstick scripts.
@@ -102,7 +114,8 @@ The following Ignite benchmark properties can be defined in the benchmark config
* `-b <num>` or `--backups <num>` - Number of backups for every key
* `-cfg <path>` or `--Config <path>` - Path to Ignite configuration file
* `-cs` or `--cacheStore` - Enable or disable cache store readThrough, writeThrough
-* `-cl` or `--client` - Client flag
+* `-cl` or `--client` - Client flag. Use this flag if you running more than one `DRIVER`, otherwise additional drivers
+would behave like a `servers`.
* `-nc` or `--nearCache` - Near cache flag
* `-nn <num>` or `--nodeNumber <num>` - Number of nodes (automatically set in `benchmark.properties`), used to wait for
the specified number of nodes to start
@@ -141,4 +154,4 @@ Use Ignite Apache JIRA (https://issues.apache.org/jira/browse/IGNITE) to file bu
License
=======
-Yardstick Ignite is available under Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0.html) Open Source license.
+Yardstick Ignite is available under Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0.html) Open Source license.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-atomic-win.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-atomic-win.properties b/modules/yardstick/config/benchmark-atomic-win.properties
index efcb0a4..202306e 100644
--- a/modules/yardstick/config/benchmark-atomic-win.properties
+++ b/modules/yardstick/config/benchmark-atomic-win.properties
@@ -51,7 +51,7 @@ set DRIVER_HOSTS=localhost
:: set REMOTE_USER=
:: Run configuration which contains all benchmarks.
-:: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+:: Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
set CONFIGS=^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetBenchmark -sn IgniteNode -ds atomic-get-1-backup,^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetOffHeapBenchmark -sn IgniteNode -ds atomic-get-offheap-1-backup,^
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-atomic.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-atomic.properties b/modules/yardstick/config/benchmark-atomic.properties
index fc827e0..7226384 100644
--- a/modules/yardstick/config/benchmark-atomic.properties
+++ b/modules/yardstick/config/benchmark-atomic.properties
@@ -55,16 +55,31 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetBenchmark -sn IgniteNode -ds atomic-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetOffHeapBenchmark -sn IgniteNode -ds atomic-get-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetOffHeapValuesBenchmark -sn IgniteNode -ds atomic-get-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds atomic-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds atomic-put-get-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds atomic-put-get-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds atomic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds atomic-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds atomic-put-offheap-val-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetBenchmark -sn IgniteNode -ds atomic-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetOffHeapBenchmark -sn IgniteNode -ds atomic-get-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetOffHeapValuesBenchmark -sn IgniteNode -ds atomic-get-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds atomic-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds atomic-put-get-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds atomic-put-get-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds atomic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds atomic-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds atomic-put-offheap-val-${b}-backup\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-bin-identity.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-bin-identity.properties b/modules/yardstick/config/benchmark-bin-identity.properties
index 3d754c5..6468d8f 100644
--- a/modules/yardstick/config/benchmark-bin-identity.properties
+++ b/modules/yardstick/config/benchmark-bin-identity.properties
@@ -24,7 +24,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -48,7 +47,7 @@ BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
# Packages where the specified benchmark is searched by reflection mechanism.
BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
-# Restart servers for each benchmark.
+# Flag which indicates to restart the servers before every benchmark execution.
RESTART_SERVERS=true
# Probe point writer class name.
@@ -81,14 +80,11 @@ t=64
# Sync mode.
sm=PRIMARY_SYNC
-# Jobs.
-j=10
-
# Run configuration which contains all benchmarks.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFieldsBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}fields-bin-id-atomic-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFieldsBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}fields-bin-id-atomic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-put-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFieldsBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}fields-bin-id-atomic-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFieldsBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}fields-bin-id-atomic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-put-${b}-backup\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-cache-load-win.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-cache-load-win.properties b/modules/yardstick/config/benchmark-cache-load-win.properties
index efe24f5..3ee11a3 100644
--- a/modules/yardstick/config/benchmark-cache-load-win.properties
+++ b/modules/yardstick/config/benchmark-cache-load-win.properties
@@ -54,7 +54,7 @@ set DRIVER_HOSTS=localhost
set ops=PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL PUT_IF_ABSENT REPLACE SCAN_QUERY SQL_QUERY CONTINUOUS_QUERY
:: Run configuration which contains all benchmarks.
-:: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+:: Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
set CONFIGS=^
-cfg %SCRIPT_DIR%\..\config\ignite-cache-load-config.xml -r 10000 -pa 500 -bs 50 -b 1 -w 20 -d 30 -t 64 -sm PRIMARY_SYNC -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds cache-random-benchmark-1-backup -nn 1^
-ltqf config\queries.sql -ltops %ops%
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-cache-load.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-cache-load.properties b/modules/yardstick/config/benchmark-cache-load.properties
index 7312cb6..12e07c3 100644
--- a/modules/yardstick/config/benchmark-cache-load.properties
+++ b/modules/yardstick/config/benchmark-cache-load.properties
@@ -21,7 +21,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -79,14 +78,13 @@ pa=500000
r=1000000
# Batch size
-
bs=50
# Allow Operation
ops="PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL PUT_IF_ABSENT REPLACE SCAN_QUERY SQL_QUERY CONTINUOUS_QUERY"
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
-cfg ${SCRIPT_DIR}/../config/ignite-cache-load-config.xml -r ${r} -pa ${pa} -bs ${bs} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteCacheRandomOperationBenchmark -sn IgniteNode -ds cache-random-benchmark-${b}-backup -nn ${nodesNum} \
-ltqf ${SCRIPT_DIR}/../config/queries.sql -ltops ${ops}"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-client-mode.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-client-mode.properties b/modules/yardstick/config/benchmark-client-mode.properties
index f7c8347..0723fcb 100644
--- a/modules/yardstick/config/benchmark-client-mode.properties
+++ b/modules/yardstick/config/benchmark-client-mode.properties
@@ -24,7 +24,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -47,6 +46,7 @@ BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
# Packages where the specified benchmark is searched by reflection mechanism.
BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+# Flag which indicates to restart the servers before every benchmark execution.
RESTART_SERVERS=true
# Probe point writer class name.
@@ -64,30 +64,48 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Jobs.
+j=10
+
# Run configuration which contains all benchmarks.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds ${ver}tx-getAndPut-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteInvokeTxBenchmark -sn IgniteNode -ds ${ver}tx-invoke-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -j 10 -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -j 10 -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -j 10 -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -j 10 -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -j 10 -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 --client -sm PRIMARY_SYNC -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds ${ver}tx-getAndPut-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteInvokeTxBenchmark -sn IgniteNode -ds ${ver}tx-invoke-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -j ${j} -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -j ${j} -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -j ${j} -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -j ${j} -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -j ${j} -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} --client -sm PRIMARY_SYNC -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-${b}-backup\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-compute-win.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-compute-win.properties b/modules/yardstick/config/benchmark-compute-win.properties
index 9c60a31..a4c6f44 100644
--- a/modules/yardstick/config/benchmark-compute-win.properties
+++ b/modules/yardstick/config/benchmark-compute-win.properties
@@ -51,7 +51,7 @@ set DRIVER_HOSTS=localhost
:: set REMOTE_USER=
:: Run configuration which contains all benchmarks.
-:: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+:: Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
set CONFIGS=^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 1 -d 30 -t 1 -sm PRIMARY_SYNC -j 10 -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds affcall-compute-1-backup,^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 1 -d 30 -t 1 -sm PRIMARY_SYNC -j 10 -dn IgniteApplyBenchmark -sn IgniteNode -ds apply-compute-1-backup,^
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-compute.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-compute.properties b/modules/yardstick/config/benchmark-compute.properties
index a4816bd..66cb930 100644
--- a/modules/yardstick/config/benchmark-compute.properties
+++ b/modules/yardstick/config/benchmark-compute.properties
@@ -55,12 +55,30 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Jobs.
+j=10
+
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds affcall-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteApplyBenchmark -sn IgniteNode -ds apply-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteBroadcastBenchmark -sn IgniteNode -ds broad-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteExecuteBenchmark -sn IgniteNode -ds exec-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteRunBenchmark -sn IgniteNode -ds run-compute-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds affcall-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteApplyBenchmark -sn IgniteNode -ds apply-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteBroadcastBenchmark -sn IgniteNode -ds broad-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteExecuteBenchmark -sn IgniteNode -ds exec-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteRunBenchmark -sn IgniteNode -ds run-compute-${b}-backup\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-failover.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-failover.properties b/modules/yardstick/config/benchmark-failover.properties
index 352e2d4..4c6a12a 100644
--- a/modules/yardstick/config/benchmark-failover.properties
+++ b/modules/yardstick/config/benchmark-failover.properties
@@ -27,7 +27,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false -ea"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
- -Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -51,6 +50,7 @@ BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
# Packages where the specified benchmark is searched by reflection mechanism.
BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+# Flag which indicates to restart the servers before every benchmark execution.
RESTART_SERVERS=true
# Probe point writer class name.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-full.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-full.properties b/modules/yardstick/config/benchmark-full.properties
index da3ae9f..b64612e 100644
--- a/modules/yardstick/config/benchmark-full.properties
+++ b/modules/yardstick/config/benchmark-full.properties
@@ -24,7 +24,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
- -Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -48,6 +47,7 @@ BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
# Packages where the specified benchmark is searched by reflection mechanism.
BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+# Flag which indicates to restart the servers before every benchmark execution.
RESTART_SERVERS=true
# Probe point writer class name.
@@ -82,35 +82,35 @@ sm=PRIMARY_SYNC
# Run configuration which contains all benchmarks.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-10-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxImplicitBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-optim-repRead-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc PESSIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-pessim-repRead-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -txi SERIALIZABLE -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serial-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-10-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs-10-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-10-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-10-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-10-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-10-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutGetBatchBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-bs-10-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxImplicitBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-optim-repRead-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc PESSIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-pessim-repRead-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -txi SERIALIZABLE -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serial-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-bs-10-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-bs-10-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-bs-10-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-bs-10-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-bs-10-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-bs-10-${b}-backup,\
"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-multicast.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-multicast.properties b/modules/yardstick/config/benchmark-multicast.properties
index b1c6b31..c10d0c6 100644
--- a/modules/yardstick/config/benchmark-multicast.properties
+++ b/modules/yardstick/config/benchmark-multicast.properties
@@ -24,7 +24,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -48,7 +47,7 @@ BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
# Packages where the specified benchmark is searched by reflection mechanism.
BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
-# Restart servers for each benchmark.
+# Flag which indicates to restart the servers before every benchmark execution.
RESTART_SERVERS=true
# Probe point writer class name.
@@ -85,57 +84,57 @@ sm=PRIMARY_SYNC
j=10
# Run configuration which contains all benchmarks.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetEntryBenchmark -sn IgniteNode -ds ${ver}atomic-put-getEntry-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxImplicitBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-optim-repRead-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -dn IgnitePutGetEntryTxBenchmark -sn IgniteNode -ds ${ver}tx-optim-repRead-put-getEntry-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc PESSIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-pessim-repRead-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc PESSIMISTIC -dn IgnitePutGetEntryTxBenchmark -sn IgniteNode -ds ${ver}tx-pessim-repRead-put-getEntry-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -txi SERIALIZABLE -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serial-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -txi SERIALIZABLE -dn IgnitePutGetEntryTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serial-put-getEntry-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds ${ver}sql-merge-all-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteReplaceIndexedValue1Benchmark -sn IgniteNode -ds ${ver}replace-indexed1-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutIfAbsentIndexedValue1Benchmark -sn IgniteNode -ds ${ver}put-if-absent-indexed1-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeBenchmark -sn IgniteNode -ds ${ver}sql-merge-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeQueryBenchmark -sn IgniteNode -ds ${ver}sql-merge-query-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue1Benchmark -sn IgniteNode -ds ${ver}sql-merge-indexed1-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue2Benchmark -sn IgniteNode -ds ${ver}sql-merge-indexed2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue8Benchmark -sn IgniteNode -ds ${ver}sql-merge-indexed8-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlInsertIndexedValue1Benchmark -sn IgniteNode -ds ${ver}sql-insert-indexed1-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlInsertIndexedValue2Benchmark -sn IgniteNode -ds ${ver}sql-insert-indexed2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlInsertIndexedValue8Benchmark -sn IgniteNode -ds ${ver}sql-insert-indexed8-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -r 300000 -dn IgniteSqlDeleteBenchmark -sn IgniteNode -ds ${ver}sql-delete-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlDeleteFilteredBenchmark -sn IgniteNode -ds ${ver}sql-delete-filtered-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlUpdateBenchmark -sn IgniteNode -ds ${ver}sql-update-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlUpdateFilteredBenchmark -sn IgniteNode -ds ${ver}sql-update-filtered-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -dn IgniteGetEntriesPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getEntriesPutAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc PESSIMISTIC -dn IgniteGetEntriesPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getEntriesPutAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetEntriesPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getEntriesPutAll-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetEntryBenchmark -sn IgniteNode -ds ${ver}atomic-put-getEntry-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxImplicitBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-optim-repRead-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -dn IgnitePutGetEntryTxBenchmark -sn IgniteNode -ds ${ver}tx-optim-repRead-put-getEntry-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc PESSIMISTIC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-pessim-repRead-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc PESSIMISTIC -dn IgnitePutGetEntryTxBenchmark -sn IgniteNode -ds ${ver}tx-pessim-repRead-put-getEntry-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -txi SERIALIZABLE -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serial-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -txc OPTIMISTIC -txi SERIALIZABLE -dn IgnitePutGetEntryTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serial-put-getEntry-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-putAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgnitePutAllSerializableTxBenchmark -sn IgniteNode -ds ${ver}tx-putAllSerializable-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 100 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds ${ver}sql-merge-all-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteReplaceIndexedValue1Benchmark -sn IgniteNode -ds ${ver}replace-indexed1-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutIfAbsentIndexedValue1Benchmark -sn IgniteNode -ds ${ver}put-if-absent-indexed1-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeBenchmark -sn IgniteNode -ds ${ver}sql-merge-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeQueryBenchmark -sn IgniteNode -ds ${ver}sql-merge-query-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue1Benchmark -sn IgniteNode -ds ${ver}sql-merge-indexed1-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue2Benchmark -sn IgniteNode -ds ${ver}sql-merge-indexed2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue8Benchmark -sn IgniteNode -ds ${ver}sql-merge-indexed8-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlInsertIndexedValue1Benchmark -sn IgniteNode -ds ${ver}sql-insert-indexed1-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlInsertIndexedValue2Benchmark -sn IgniteNode -ds ${ver}sql-insert-indexed2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlInsertIndexedValue8Benchmark -sn IgniteNode -ds ${ver}sql-insert-indexed8-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -r 300000 -dn IgniteSqlDeleteBenchmark -sn IgniteNode -ds ${ver}sql-delete-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlDeleteFilteredBenchmark -sn IgniteNode -ds ${ver}sql-delete-filtered-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlUpdateBenchmark -sn IgniteNode -ds ${ver}sql-update-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlUpdateFilteredBenchmark -sn IgniteNode -ds ${ver}sql-update-filtered-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getAllPutAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -dn IgniteGetEntriesPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-optimistic-getEntriesPutAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc PESSIMISTIC -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getAllPutAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc PESSIMISTIC -dn IgniteGetEntriesPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-pessimistic-getEntriesPutAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetAllPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getAllPutAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -txc OPTIMISTIC -txi SERIALIZABLE -dn IgniteGetEntriesPutAllTxBenchmark -sn IgniteNode -ds ${ver}tx-opt-serializable-getEntriesPutAll-${b}-backup,\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-put-indexed-val.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-put-indexed-val.properties b/modules/yardstick/config/benchmark-put-indexed-val.properties
index e81ae6f..0d699c5 100644
--- a/modules/yardstick/config/benchmark-put-indexed-val.properties
+++ b/modules/yardstick/config/benchmark-put-indexed-val.properties
@@ -55,10 +55,25 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutIndexedValue1Benchmark -sn IgniteNode -ds atomic-put-indexed-val-1,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutIndexedValue2Benchmark -sn IgniteNode -ds atomic-put-indexed-val-2,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutIndexedValue8Benchmark -sn IgniteNode -ds atomic-put-indexed-val-8\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutIndexedValue1Benchmark -sn IgniteNode -ds atomic-put-indexed-val-1,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutIndexedValue2Benchmark -sn IgniteNode -ds atomic-put-indexed-val-2,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutIndexedValue8Benchmark -sn IgniteNode -ds atomic-put-indexed-val-8\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-query-put-separated.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-query-put-separated.properties b/modules/yardstick/config/benchmark-query-put-separated.properties
index 6d2ce33..500e4fa 100644
--- a/modules/yardstick/config/benchmark-query-put-separated.properties
+++ b/modules/yardstick/config/benchmark-query-put-separated.properties
@@ -24,7 +24,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -83,5 +82,5 @@ sm=PRIMARY_SYNC
# Run configuration which contains separated query put benchmark.
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w ${w} -d ${d} -t ${t} -sm PRIMARY_SYNC -dn IgniteSqlQueryPutSeparatedBenchmark -sn IgniteNode -ds ${ver}query-put-separated-1b,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b {b} -w ${w} -d ${d} -t ${t} -sm PRIMARY_SYNC -dn IgniteSqlQueryPutSeparatedBenchmark -sn IgniteNode -ds ${ver}query-put-separated-1b,\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-query-win.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-query-win.properties b/modules/yardstick/config/benchmark-query-win.properties
index b71f3e4..6bd8fb9 100644
--- a/modules/yardstick/config/benchmark-query-win.properties
+++ b/modules/yardstick/config/benchmark-query-win.properties
@@ -51,7 +51,7 @@ set DRIVER_HOSTS=localhost
:: set REMOTE_USER=
:: Run configuration which contains all benchmarks.
-:: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+:: Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
set CONFIGS=^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds sql-query-1-backup,^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds sql-query-offheap-1-backup,^
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-query.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-query.properties b/modules/yardstick/config/benchmark-query.properties
index 486d00e..af9b07e 100644
--- a/modules/yardstick/config/benchmark-query.properties
+++ b/modules/yardstick/config/benchmark-query.properties
@@ -55,15 +55,30 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds sql-query-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds sql-query-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds sql-query-join-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds sql-query-join-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds sql-query-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds sql-query-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -jdbc jdbc:ignite://127.0.0.1/query -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcSqlQueryBenchmark -sn IgniteNode -ds sql-query-jdbc-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -jdbc jdbc:ignite:cfg://cache=query@config/ignite-jdbc-config.xml -t 64 -sm PRIMARY_SYNC -dn IgniteJdbcSqlQueryBenchmark -sn IgniteNode -ds sql-query-jdbc-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds sql-query-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds sql-query-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds sql-query-join-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds sql-query-join-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds sql-query-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds sql-query-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -jdbc jdbc:ignite://127.0.0.1/query -t ${t} -sm ${sm} -dn IgniteJdbcSqlQueryBenchmark -sn IgniteNode -ds sql-query-jdbc-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -jdbc jdbc:ignite:cfg://cache=query@config/ignite-jdbc-config.xml -t ${t} -sm ${sm} -dn IgniteJdbcSqlQueryBenchmark -sn IgniteNode -ds sql-query-jdbc-${b}-backup\
"
[47/50] [abbrv] ignite git commit: Release Notes
Posted by av...@apache.org.
Release Notes
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a70f5c4c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a70f5c4c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a70f5c4c
Branch: refs/heads/master
Commit: a70f5c4cece805b7a8c96bdb9fa763b316169a12
Parents: a779c5f
Author: Anton Vinogradov <av...@apache.org>
Authored: Mon Feb 27 17:35:17 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Mon Feb 27 17:35:17 2017 +0300
----------------------------------------------------------------------
RELEASE_NOTES.txt | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a70f5c4c/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index ea88455..f46bfe6 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,6 +1,27 @@
Apache Ignite Release Notes
===========================
+Apache Ignite In-Memory Data Fabric 1.9
+---------------------------------------
+Ignite:
+* Added Data streamer mode for DML
+* Added Discovery SPI Implementation for Ignite Kubernetes Pods
+* SQL: Query can utilize multiple threads
+* SQL: Sorted merge index supported
+* SQL: Improved distributed SQL support
+* Benchmarking simplified and automated
+* Fixed licenses generation during build
+* ignite-spark module upgraded to Spark 2.0
+
+Ignite.NET:
+* DML support
+* TransactionScope API for Ignite transactions support
+
+Ignite CPP:
+* DML support
+* Implemented LoadCache
+* ContinuousQuery support
+
Apache Ignite In-Memory Data Fabric 1.8
---------------------------------------
Ignite:
[39/50] [abbrv] ignite git commit: IGNITE-4106: Improved
documentation.
Posted by av...@apache.org.
IGNITE-4106: Improved documentation.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/94dab7bb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/94dab7bb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/94dab7bb
Branch: refs/heads/master
Commit: 94dab7bb4d8a3c422029e8bece4c0655c7c7c6af
Parents: 2d0d567
Author: devozerov <vo...@gridgain.com>
Authored: Wed Feb 22 15:39:19 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Feb 22 15:39:19 2017 +0300
----------------------------------------------------------------------
.../configuration/CacheConfiguration.java | 27 ++++++++------------
1 file changed, 10 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/94dab7bb/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
index 149f25a..e160ba9 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
@@ -223,8 +223,8 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
/** Default threshold for concurrent loading of keys from {@link CacheStore}. */
public static final int DFLT_CONCURRENT_LOAD_ALL_THRESHOLD = 5;
- /** Default SQL query parallelism level */
- public static final int DFLT_SQL_QUERY_PARALLELISM_LVL = 1;
+ /** Default query parallelism. */
+ public static final int DFLT_QUERY_PARALLELISM = 1;
/** Cache name. */
private String name;
@@ -414,7 +414,7 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
private Collection<QueryEntity> qryEntities;
/** */
- private int qryParallelism = DFLT_SQL_QUERY_PARALLELISM_LVL;
+ private int qryParallelism = DFLT_QUERY_PARALLELISM;
/** Empty constructor (all values are initialized to their defaults). */
public CacheConfiguration() {
@@ -2121,32 +2121,25 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
* <p>
* As of {@code Apache Ignite 1.9} this hint is only supported for SQL queries with the following restrictions:
* <ul>
- * <li>Hint cannot be used for {@code REPLICATED} cache, exception is thrown otherwise</li>
* <li>All caches participating in query must have the same degree of parallelism, exception is thrown
* otherwise</li>
+ * <li>All queries on the given cache will follow provided degree of parallelism</li>
* </ul>
* These restrictions will be removed in future versions of Apache Ignite.
* <p>
- * Defaults to {@code 1}.
+ * Defaults to {@link #DFLT_QUERY_PARALLELISM}.
+ *
+ * @return Query parallelism.
*/
public int getQueryParallelism() {
return qryParallelism;
}
/**
- * Defines a hint to query execution engine on desired degree of parallelism within a single node.
- * Query executor may or may not use this hint depending on estimated query costs. Query executor may define
- * certain restrictions on parallelism depending on query type and/or cache type.
- * <p>
- * As of {@code Apache Ignite 1.9} this hint is only supported for SQL queries with the following restrictions:
- * <ul>
- * <li>Hint cannot be used for {@code REPLICATED} cache, exception is thrown otherwise</li>
- * <li>All caches participating in query must have the same degree of parallelism, exception is thrown
- * otherwise</li>
- * </ul>
- * These restrictions will be removed in future versions of Apache Ignite.
+ * Sets query parallelism.
*
- * @param qryParallelism Query parallelizm level.
+ * @param qryParallelism Query parallelism.
+ * @see #getQueryParallelism()
* @return {@code this} for chaining.
*/
public CacheConfiguration<K,V> setQueryParallelism(int qryParallelism) {
[49/50] [abbrv] ignite git commit: Release Notes
Posted by av...@apache.org.
Release Notes
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bea863f4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bea863f4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bea863f4
Branch: refs/heads/master
Commit: bea863f4427651cbcd67ffc20e22cc8d97e3bbf8
Parents: b7949c4
Author: Anton Vinogradov <av...@apache.org>
Authored: Tue Feb 28 16:43:49 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Tue Feb 28 16:43:49 2017 +0300
----------------------------------------------------------------------
RELEASE_NOTES.txt | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bea863f4/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index f46bfe6..65cc9d4 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -7,7 +7,6 @@ Ignite:
* Added Data streamer mode for DML
* Added Discovery SPI Implementation for Ignite Kubernetes Pods
* SQL: Query can utilize multiple threads
-* SQL: Sorted merge index supported
* SQL: Improved distributed SQL support
* Benchmarking simplified and automated
* Fixed licenses generation during build
[44/50] [abbrv] ignite git commit: Merge branch 'ignite-1.9' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-1.9
Posted by av...@apache.org.
Merge branch 'ignite-1.9' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-1.9
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/57291773
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/57291773
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/57291773
Branch: refs/heads/master
Commit: 57291773cf9ddfbf78c3370429a1fb630a907603
Parents: cf3de19 80352d0
Author: Sergi Vladykin <se...@gmail.com>
Authored: Wed Feb 22 16:43:21 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Wed Feb 22 16:43:21 2017 +0300
----------------------------------------------------------------------
.../cpp/common/project/vs/common.vcxproj | 1 -
.../common/project/vs/common.vcxproj.filters | 6 -----
.../platforms/cpp/common/project/vs/targetver.h | 25 --------------------
.../platforms/cpp/jni/project/vs/jni.vcxproj | 1 -
.../cpp/jni/project/vs/jni.vcxproj.filters | 3 ---
.../platforms/cpp/jni/project/vs/targetver.h | 25 --------------------
6 files changed, 61 deletions(-)
----------------------------------------------------------------------
[48/50] [abbrv] ignite git commit: JDK 8 compilation fix
Posted by av...@apache.org.
JDK 8 compilation fix
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b7949c4f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b7949c4f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b7949c4f
Branch: refs/heads/master
Commit: b7949c4fe2cc4e27035e0b6576c361cd21246432
Parents: a70f5c4
Author: Anton Vinogradov <av...@apache.org>
Authored: Tue Feb 28 16:29:28 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Tue Feb 28 16:29:28 2017 +0300
----------------------------------------------------------------------
parent/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7949c4f/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index a4972d1..a17732d 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -937,7 +937,7 @@
<jdk>[1.8,)</jdk>
</activation>
<properties>
- <javadoc.opts>-Xdoclint:none</javadoc.opts>
+ <javadoc.opts>-Xdoclint:none --allow-script-in-comments</javadoc.opts>
</properties>
<build>
<plugins>
[41/50] [abbrv] ignite git commit: IGNITE-4745: CPP: removed unused
targetver.h files. This closes #1570.
Posted by av...@apache.org.
IGNITE-4745: CPP: removed unused targetver.h files. This closes #1570.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7be2a70e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7be2a70e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7be2a70e
Branch: refs/heads/master
Commit: 7be2a70e49febfbf2642d54420f5325f059ef327
Parents: 94dab7b
Author: isapego <ig...@gmail.com>
Authored: Wed Feb 22 16:35:55 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Feb 22 16:35:55 2017 +0300
----------------------------------------------------------------------
.../cpp/common/project/vs/common.vcxproj | 1 -
.../common/project/vs/common.vcxproj.filters | 6 -----
.../platforms/cpp/common/project/vs/targetver.h | 25 --------------------
.../platforms/cpp/jni/project/vs/jni.vcxproj | 1 -
.../cpp/jni/project/vs/jni.vcxproj.filters | 3 ---
.../platforms/cpp/jni/project/vs/targetver.h | 25 --------------------
6 files changed, 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7be2a70e/modules/platforms/cpp/common/project/vs/common.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/common/project/vs/common.vcxproj b/modules/platforms/cpp/common/project/vs/common.vcxproj
index 7a57ac0..73a0fef 100644
--- a/modules/platforms/cpp/common/project/vs/common.vcxproj
+++ b/modules/platforms/cpp/common/project/vs/common.vcxproj
@@ -182,7 +182,6 @@
<ClInclude Include="..\..\include\ignite\timestamp.h" />
<ClInclude Include="..\..\os\win\include\ignite\common\common.h" />
<ClInclude Include="..\..\os\win\include\ignite\common\concurrent_os.h" />
- <ClInclude Include="targetver.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\os\win\src\common\concurrent_os.cpp" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/7be2a70e/modules/platforms/cpp/common/project/vs/common.vcxproj.filters
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/common/project/vs/common.vcxproj.filters b/modules/platforms/cpp/common/project/vs/common.vcxproj.filters
index f08471e..49a63d4 100644
--- a/modules/platforms/cpp/common/project/vs/common.vcxproj.filters
+++ b/modules/platforms/cpp/common/project/vs/common.vcxproj.filters
@@ -1,9 +1,6 @@
\ufeff<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
- <Filter Include="Misc">
- <UniqueIdentifier>{1dbec2be-5cb4-4f70-aef6-b4627d39b99b}</UniqueIdentifier>
- </Filter>
<Filter Include="Code">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
@@ -13,9 +10,6 @@
</Filter>
</ItemGroup>
<ItemGroup>
- <ClInclude Include="targetver.h">
- <Filter>Misc</Filter>
- </ClInclude>
<ClInclude Include="..\..\os\win\include\ignite\common\common.h">
<Filter>Code\common</Filter>
</ClInclude>
http://git-wip-us.apache.org/repos/asf/ignite/blob/7be2a70e/modules/platforms/cpp/common/project/vs/targetver.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/common/project/vs/targetver.h b/modules/platforms/cpp/common/project/vs/targetver.h
deleted file mode 100644
index 4bea158..0000000
--- a/modules/platforms/cpp/common/project/vs/targetver.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// Including SDKDDKVer.h defines the highest available Windows platform.
-
-// If you wish to build your application for a previous Windows platform, include WinSDKVer.h and
-// set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h.
-
-#include <SDKDDKVer.h>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/7be2a70e/modules/platforms/cpp/jni/project/vs/jni.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/jni/project/vs/jni.vcxproj b/modules/platforms/cpp/jni/project/vs/jni.vcxproj
index cb03197..310ee18 100644
--- a/modules/platforms/cpp/jni/project/vs/jni.vcxproj
+++ b/modules/platforms/cpp/jni/project/vs/jni.vcxproj
@@ -184,7 +184,6 @@
<ClInclude Include="..\..\include\ignite\jni\exports.h" />
<ClInclude Include="..\..\include\ignite\jni\java.h" />
<ClInclude Include="..\..\include\ignite\jni\utils.h" />
- <ClInclude Include="targetver.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\os\win\src\utils.cpp" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/7be2a70e/modules/platforms/cpp/jni/project/vs/jni.vcxproj.filters
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/jni/project/vs/jni.vcxproj.filters b/modules/platforms/cpp/jni/project/vs/jni.vcxproj.filters
index 7bc8dfd..93fa33d 100644
--- a/modules/platforms/cpp/jni/project/vs/jni.vcxproj.filters
+++ b/modules/platforms/cpp/jni/project/vs/jni.vcxproj.filters
@@ -16,9 +16,6 @@
<ClInclude Include="..\..\include\ignite\jni\java.h">
<Filter>Code</Filter>
</ClInclude>
- <ClInclude Include="targetver.h">
- <Filter>Misc</Filter>
- </ClInclude>
<ClInclude Include="..\..\include\ignite\jni\utils.h">
<Filter>Code</Filter>
</ClInclude>
http://git-wip-us.apache.org/repos/asf/ignite/blob/7be2a70e/modules/platforms/cpp/jni/project/vs/targetver.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/jni/project/vs/targetver.h b/modules/platforms/cpp/jni/project/vs/targetver.h
deleted file mode 100644
index 4bea158..0000000
--- a/modules/platforms/cpp/jni/project/vs/targetver.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-// Including SDKDDKVer.h defines the highest available Windows platform.
-
-// If you wish to build your application for a previous Windows platform, include WinSDKVer.h and
-// set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h.
-
-#include <SDKDDKVer.h>
\ No newline at end of file
[50/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/ignite-1.9'
Posted by av...@apache.org.
Merge remote-tracking branch 'remotes/origin/ignite-1.9'
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
# modules/kubernetes/config/Dockerfile
# modules/kubernetes/config/ignite-deployment.yaml
# modules/kubernetes/config/ignite-service.yaml
# pom.xml
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/cc6257f8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/cc6257f8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/cc6257f8
Branch: refs/heads/master
Commit: cc6257f8c53da5f6da32b0f184065c1e35a75acf
Parents: 5456caf bea863f
Author: Anton Vinogradov <av...@apache.org>
Authored: Wed Mar 1 17:32:17 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Mar 1 17:32:17 2017 +0300
----------------------------------------------------------------------
RELEASE_NOTES.txt | 20 +
assembly/dependencies-hadoop.xml | 3 +
assembly/release-base.xml | 5 -
assembly/release-fabric-base.xml | 5 +
assembly/release-sources.xml | 3 +-
examples/pom-standalone-lgpl.xml | 6 +
examples/pom-standalone.xml | 6 +
modules/clients/src/test/config/jdbc-config.xml | 2 +-
.../jdbc2/JdbcAbstractDmlStatementSelfTest.java | 49 +-
.../jdbc2/JdbcInsertStatementSelfTest.java | 51 +
.../jdbc2/JdbcMergeStatementSelfTest.java | 51 +
.../internal/jdbc2/JdbcStreamingSelfTest.java | 187 ++
.../jdbc2/JdbcUpdateStatementSelfTest.java | 50 +
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 1 +
.../org/apache/ignite/IgniteJdbcDriver.java | 30 +
.../apache/ignite/IgniteSystemProperties.java | 38 +
.../configuration/CacheConfiguration.java | 41 +
.../ignite/internal/binary/BinaryContext.java | 3 +
.../ignite/internal/binary/BinaryTreeMap.java | 6 +-
.../ignite/internal/jdbc2/JdbcConnection.java | 72 +-
.../internal/jdbc2/JdbcPreparedStatement.java | 34 +-
.../ignite/internal/jdbc2/JdbcStatement.java | 20 +-
.../jdbc2/JdbcStreamedPreparedStatement.java | 59 +
.../processors/cache/EntryGetResult.java | 40 +-
.../processors/cache/EntryGetWithTtlResult.java | 58 +
.../processors/cache/GridCacheAdapter.java | 69 +-
.../processors/cache/GridCacheContext.java | 122 +-
.../processors/cache/GridCacheEntryEx.java | 2 +-
.../processors/cache/GridCacheMapEntry.java | 29 +-
.../processors/cache/GridCacheProcessor.java | 3 +
.../processors/cache/IgniteCacheProxy.java | 6 +-
.../distributed/dht/GridDhtCacheAdapter.java | 4 +-
.../cache/distributed/dht/GridDhtGetFuture.java | 24 +-
.../distributed/dht/GridDhtGetSingleFuture.java | 24 +-
.../dht/GridPartitionedGetFuture.java | 19 +-
.../dht/GridPartitionedSingleGetFuture.java | 7 +-
.../dht/atomic/GridDhtAtomicCache.java | 12 +-
.../dht/colocated/GridDhtColocatedCache.java | 15 +-
.../distributed/near/GridNearGetFuture.java | 9 +-
.../cache/distributed/near/GridNearTxLocal.java | 8 +-
.../local/atomic/GridLocalAtomicCache.java | 13 +-
.../cache/query/GridCacheSqlQuery.java | 45 +-
.../transactions/IgniteTxLocalAdapter.java | 65 +-
.../closure/GridClosureProcessor.java | 2 +-
.../processors/cluster/GridUpdateNotifier.java | 2 +-
.../processors/hadoop/HadoopJobProperty.java | 2 +-
.../processors/query/GridQueryIndexing.java | 58 +-
.../processors/query/GridQueryProcessor.java | 194 +-
.../messages/GridQueryNextPageRequest.java | 29 +-
.../messages/GridQueryNextPageResponse.java | 29 +-
.../resources/META-INF/classnames.properties | 6 +-
.../cache/GridCacheAbstractFullApiSelfTest.java | 20 +-
.../processors/cache/GridCacheTestEntryEx.java | 2 +-
.../CacheLockReleaseNodeLeaveTest.java | 64 +-
.../IgniteCacheExpiryPolicyAbstractTest.java | 42 +
.../tcp/TcpCommunicationSpiDropNodesTest.java | 3 +-
.../TcpCommunicationSpiFaultyClientTest.java | 3 +-
.../query/h2/opt/GridH2SpatialIndex.java | 56 +-
.../query/h2/GridH2IndexingGeoSelfTest.java | 409 +++--
.../h2/GridH2IndexingSegmentedGeoSelfTest.java | 37 +
.../testsuites/GeoSpatialIndexingTestSuite.java | 2 +
.../hadoop/shuffle/HadoopShuffleJob.java | 3 +-
.../cache/query/GridCacheTwoStepQuery.java | 17 +
.../query/h2/DmlStatementsProcessor.java | 267 ++-
.../processors/query/h2/IgniteH2Indexing.java | 406 ++++-
.../query/h2/dml/UpdatePlanBuilder.java | 2 +-
.../query/h2/opt/DistributedJoinMode.java | 51 +
.../query/h2/opt/GridH2CollocationModel.java | 78 +-
.../query/h2/opt/GridH2IndexBase.java | 348 +++-
.../query/h2/opt/GridH2QueryContext.java | 84 +-
.../query/h2/opt/GridH2RowDescriptor.java | 5 -
.../query/h2/opt/GridH2TreeIndex.java | 151 +-
.../processors/query/h2/sql/DmlAstUtils.java | 54 +-
.../processors/query/h2/sql/GridSqlAlias.java | 13 +-
.../processors/query/h2/sql/GridSqlArray.java | 8 +-
.../processors/query/h2/sql/GridSqlAst.java | 61 +
.../processors/query/h2/sql/GridSqlColumn.java | 85 +-
.../processors/query/h2/sql/GridSqlConst.java | 6 +-
.../processors/query/h2/sql/GridSqlElement.java | 43 +-
.../query/h2/sql/GridSqlFunction.java | 18 +-
.../processors/query/h2/sql/GridSqlJoin.java | 35 +-
.../processors/query/h2/sql/GridSqlKeyword.java | 3 +-
.../query/h2/sql/GridSqlOperation.java | 6 +-
.../query/h2/sql/GridSqlOperationType.java | 4 +-
.../query/h2/sql/GridSqlParameter.java | 4 +-
.../query/h2/sql/GridSqlPlaceholder.java | 2 +-
.../processors/query/h2/sql/GridSqlQuery.java | 80 +-
.../query/h2/sql/GridSqlQueryParser.java | 246 ++-
.../query/h2/sql/GridSqlQuerySplitter.java | 1656 +++++++++++++++---
.../processors/query/h2/sql/GridSqlSelect.java | 121 +-
.../query/h2/sql/GridSqlStatement.java | 11 +-
.../query/h2/sql/GridSqlSubquery.java | 31 +-
.../processors/query/h2/sql/GridSqlTable.java | 19 +-
.../processors/query/h2/sql/GridSqlType.java | 6 +-
.../processors/query/h2/sql/GridSqlUnion.java | 66 +-
.../processors/query/h2/sql/GridSqlValue.java | 25 -
.../query/h2/twostep/GridMapQueryExecutor.java | 234 ++-
.../query/h2/twostep/GridMergeIndex.java | 457 ++++-
.../query/h2/twostep/GridMergeIndexSorted.java | 284 +++
.../h2/twostep/GridMergeIndexUnsorted.java | 40 +-
.../h2/twostep/GridReduceQueryExecutor.java | 120 +-
.../h2/twostep/msg/GridH2IndexRangeRequest.java | 60 +-
.../twostep/msg/GridH2IndexRangeResponse.java | 62 +-
.../h2/twostep/msg/GridH2QueryRequest.java | 12 +
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 2 +-
...niteCacheAbstractInsertSqlQuerySelfTest.java | 6 +-
.../IgniteCacheInsertSqlQuerySelfTest.java | 14 +
.../IgniteCacheUpdateSqlQuerySelfTest.java | 11 +-
.../query/IgniteSqlSegmentedIndexSelfTest.java | 263 +++
.../query/IgniteSqlSplitterSelfTest.java | 237 ++-
.../h2/GridIndexingSpiAbstractSelfTest.java | 26 +-
.../h2/sql/AbstractH2CompareQueryTest.java | 21 +-
.../query/h2/sql/GridQueryParsingTest.java | 72 +-
.../query/h2/sql/H2CompareBigQueryTest.java | 56 +-
.../FetchingQueryCursorStressTest.java | 277 +++
.../IgniteCacheQuerySelfTestSuite.java | 2 +
.../processors/query/h2/sql/bigQuery.sql | 2 +-
modules/kubernetes/config/Dockerfile | 17 +
.../kubernetes/config/ignite-deployment.yaml | 15 +
modules/kubernetes/config/ignite-service.yaml | 15 +
.../cpp/common/project/vs/common.vcxproj | 1 -
.../common/project/vs/common.vcxproj.filters | 6 -
.../platforms/cpp/common/project/vs/targetver.h | 25 -
modules/platforms/cpp/core-test/Makefile.am | 1 +
.../cpp/core-test/config/cache-store.xml | 71 +
.../cpp/core-test/project/vs/core-test.vcxproj | 5 +
.../project/vs/core-test.vcxproj.filters | 9 +
.../src/binary_identity_resolver_test.cpp | 2 +-
.../src/binary_reader_writer_raw_test.cpp | 2 +-
.../core-test/src/binary_reader_writer_test.cpp | 2 +-
.../cpp/core-test/src/binary_session_test.cpp | 2 +-
.../cpp/core-test/src/cache_query_test.cpp | 2 +-
.../cpp/core-test/src/cache_store_test.cpp | 151 ++
.../cpp/core-test/src/continuous_query_test.cpp | 2 +-
.../cpp/core-test/src/handle_registry_test.cpp | 2 +-
.../cpp/core-test/src/ignite_error_test.cpp | 2 +-
.../cpp/core-test/src/ignition_test.cpp | 2 +-
.../cpp/core-test/src/interop_memory_test.cpp | 2 +-
.../cpp/core-test/src/reference_test.cpp | 2 +-
.../platforms/cpp/core-test/src/test_utils.cpp | 7 +
.../cpp/core-test/src/transactions_test.cpp | 2 +-
.../cpp/core/include/ignite/cache/cache.h | 54 +-
.../cache/query/continuous/continuous_query.h | 2 +-
.../core/include/ignite/impl/cache/cache_impl.h | 19 +
.../ignite/impl/interop/interop_target.h | 25 +
.../cpp/core/src/impl/cache/cache_impl.cpp | 48 +
.../core/src/impl/interop/interop_target.cpp | 36 +-
.../platforms/cpp/jni/project/vs/jni.vcxproj | 1 -
.../cpp/jni/project/vs/jni.vcxproj.filters | 3 -
.../platforms/cpp/jni/project/vs/targetver.h | 25 -
.../Properties/AssemblyInfo.cs | 2 +-
.../Properties/AssemblyInfo.cs | 2 +-
.../Properties/AssemblyInfo.cs | 2 +-
.../Properties/AssemblyInfo.cs | 2 +-
.../Properties/AssemblyInfo.cs | 2 +-
.../Properties/AssemblyInfo.cs | 2 +-
modules/yardstick/config/ignite-base-config.xml | 25 +
.../yardstick/cache/IgniteGetAllBenchmark.java | 42 +
.../cache/IgniteGetAllOffHeapBenchmark.java | 30 +
.../yardstick/cache/IgniteGetBenchmark.java | 35 +
.../cache/IgnitePutAllOffHeapBenchmark.java | 30 +
.../cache/IgnitePutAllTxOffHeapBenchmark.java | 30 +
.../IgnitePutOffHeapIndexedValue8Benchmark.java | 30 +
.../IgnitePutRandomValueSizeBenchmark.java | 41 +
parent/pom.xml | 4 +-
165 files changed, 7532 insertions(+), 1856 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cc6257f8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cc6257f8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cc6257f8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cc6257f8/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index e4b0c1f,b4bf608..247b5a6
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@@ -91,7 -94,9 +94,8 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.query.GridQueryIndexing;
import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
-import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
+ import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2DefaultTableEngine;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOffheap;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap;
@@@ -1115,15 -1279,16 +1278,21 @@@ public class IgniteH2Indexing implement
setupConnection(conn, false, false);
- GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
+ GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter)
+ .distributedJoinMode(OFF));
+
+ GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
+ U.currentTimeMillis(), null, true);
+
+ runs.put(run.id(), run);
+ GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
+ U.currentTimeMillis(), null, true);
+
+ runs.put(run.id(), run);
+
try {
- ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
+ ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, cancel);
return new KeyValIterator(rs);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cc6257f8/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index f54fab6,4cae6ac..a419c24
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@@ -62,12 -62,11 +62,12 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
import org.apache.ignite.internal.processors.query.GridQueryCancel;
+ import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext;
- import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlType;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse;
[21/50] [abbrv] ignite git commit: IGNITE-4372 Ids quoting fix
Posted by av...@apache.org.
IGNITE-4372 Ids quoting fix
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/14290d3f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/14290d3f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/14290d3f
Branch: refs/heads/master
Commit: 14290d3fe44af7e2269d42361f2bb1207b0ee8c3
Parents: 166e65c
Author: Alexander Paschenko <al...@gmail.com>
Authored: Tue Feb 21 14:00:45 2017 +0300
Committer: Alexander Paschenko <al...@gmail.com>
Committed: Tue Feb 21 14:00:45 2017 +0300
----------------------------------------------------------------------
.../internal/processors/query/h2/sql/DmlAstUtils.java | 10 ++++++----
.../internal/processors/query/h2/sql/GridSqlFunction.java | 2 +-
2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/14290d3f/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
index 35f0983..f96e5e3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
@@ -25,6 +25,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.dml.FastUpdateArgument;
import org.apache.ignite.internal.processors.query.h2.dml.FastUpdateArguments;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
@@ -33,6 +34,7 @@ import org.apache.ignite.internal.util.lang.IgnitePair;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
+import org.h2.command.Parser;
import org.h2.expression.Expression;
import org.h2.table.Column;
import org.h2.table.Table;
@@ -47,8 +49,6 @@ import org.h2.value.ValueTimestamp;
import org.h2.value.ValueTimestampUtc;
import org.jetbrains.annotations.Nullable;
-import org.apache.ignite.internal.processors.query.h2.dml.FastUpdateArgument;
-
/**
* AST utils for DML
*/
@@ -85,7 +85,7 @@ public final class DmlAstUtils {
for (int i = 0; i < cols.length; i++) {
GridSqlArray arr = new GridSqlArray(rows.size());
- String colName = IgniteH2Indexing.escapeName(cols[i].columnName(), desc.quoteAllIdentifiers());
+ String colName = cols[i].columnName();
GridSqlAlias alias = new GridSqlAlias(colName, arr);
@@ -109,6 +109,8 @@ public final class DmlAstUtils {
args[i].addChild(row[i]);
}
+ sel.getSQL();
+
return sel;
}
else {
@@ -339,7 +341,7 @@ public final class DmlAstUtils {
mapQry.addColumn(valCol, true);
for (GridSqlColumn c : update.cols()) {
- String newColName = "_upd_" + c.columnName();
+ String newColName = Parser.quoteIdentifier("_upd_" + c.columnName());
// We have to use aliases to cover cases when the user
// wants to update _val field directly (if it's a literal)
GridSqlAlias alias = new GridSqlAlias(newColName, elementOrDefault(update.set().get(c.columnName()), c), true);
http://git-wip-us.apache.org/repos/asf/ignite/blob/14290d3f/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
index f194b07..46a558a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
@@ -143,7 +143,7 @@ public class GridSqlFunction extends GridSqlElement {
GridSqlElement e = child(i);
// id int = ?, name varchar = ('aaa', 'bbb')
- buff.append(((GridSqlAlias)e).alias())
+ buff.append(Parser.quoteIdentifier(((GridSqlAlias)e).alias()))
.append(' ')
.append(e.resultType().sql())
.append('=')
[02/50] [abbrv] ignite git commit: IGNITE-4212 Ignite Benchmarking
Simplification and Automation / IGNITE-4478 Fixing documentation and
resolving lack of usability (cherry picked from commit 1265793)
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-remote-sample.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-remote-sample.properties b/modules/yardstick/config/benchmark-remote-sample.properties
new file mode 100644
index 0000000..5f158ac
--- /dev/null
+++ b/modules/yardstick/config/benchmark-remote-sample.properties
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Contains benchmarks for ATOMIC cache.
+#
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+# JVM_OPTS=${JVM_OPTS}" \
+# -XX:+UseParNewGC \
+# -XX:+UseConcMarkSweepGC \
+# -XX:+UseTLAB \
+# -XX:NewSize=128m \
+# -XX:MaxNewSize=128m \
+# -XX:MaxTenuringThreshold=0 \
+# -XX:SurvivorRatio=1024 \
+# -XX:+UseCMSInitiatingOccupancyOnly \
+# -XX:CMSInitiatingOccupancyFraction=60 \
+#"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# Comma-separated list of the hosts to run BenchmarkServers on. 2 nodes on local host are enabled by default.
+SERVER_HOSTS=localhost,localhost
+
+# Comma-separated list of the hosts to run BenchmarkDrivers on. 1 node on local host is enabled by default.
+DRIVER_HOSTS=localhost
+
+# Flag which indicates to upload yardstick from one host to every other and collect logs and results automatically.
+AUTO_COPY=true
+
+# Remote username.
+# REMOTE_USER=
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Run configuration.
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds atomic-put-${b}-backup,\
+"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-remote.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-remote.properties b/modules/yardstick/config/benchmark-remote.properties
new file mode 100644
index 0000000..0cda79e
--- /dev/null
+++ b/modules/yardstick/config/benchmark-remote.properties
@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Contains all benchmarks
+#
+
+now0=`date +'%H%M%S'`
+
+# JVM options.
+JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
+
+# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
+JVM_OPTS=${JVM_OPTS}" \
+ -XX:+PrintGCDetails \
+ -verbose:gc \
+ -XX:+UseParNewGC \
+ -XX:+UseConcMarkSweepGC \
+ -XX:+UseTLAB \
+ -XX:NewSize=128m \
+ -XX:MaxNewSize=128m \
+ -XX:MaxTenuringThreshold=0 \
+ -XX:SurvivorRatio=1024 \
+ -XX:+UseCMSInitiatingOccupancyOnly \
+ -XX:CMSInitiatingOccupancyFraction=60 \
+"
+
+#Ignite version
+ver="RELEASE-"
+
+# List of default probes.
+# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux).
+BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
+
+# Packages where the specified benchmark is searched by reflection mechanism.
+BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+
+# Flag which indicates to restart the servers after execution of every benchmark.
+RESTART_SERVERS=true
+
+# Probe point writer class name.
+# BENCHMARK_WRITER=
+
+# Comma-separated list of the hosts to run BenchmarkServers on. 2 nodes on local host are enabled by default.
+SERVER_HOSTS=localhost,localhost
+
+# Comma-separated list of the hosts to run BenchmarkDrivers on. 1 node on local host is enabled by default.
+DRIVER_HOSTS=localhost
+
+# Remote username.
+# REMOTE_USER=
+
+# Flag which indicates to upload yardstick from one host to every other and collect logs and results automatically.
+AUTO_COPY=true
+
+# Number of nodes, used to wait for the specified number of nodes to start.
+nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Jobs.
+j=10
+
+# Run configuration which contains all benchmarks.
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
+CONFIGS="\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds ${ver}tx-getAndPut-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteInvokeTxBenchmark -sn IgniteNode -ds ${ver}tx-invoke-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-${b}-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -sm PRIMARY_SYNC -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -col -sm PRIMARY_SYNC -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-collocated-putAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -sm PRIMARY_SYNC -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-tx-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-remote-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -col -sm PRIMARY_SYNC -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}atomic-collocated-putAll-tx-${b}-backup\
+"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-sample.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-sample.properties b/modules/yardstick/config/benchmark-sample.properties
index fee7cbb..1932ae7 100644
--- a/modules/yardstick/config/benchmark-sample.properties
+++ b/modules/yardstick/config/benchmark-sample.properties
@@ -55,8 +55,26 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Jobs.
+j=10
+
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds atomic-put-1-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds atomic-put-${b}-backup,\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-sql-dml.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-sql-dml.properties b/modules/yardstick/config/benchmark-sql-dml.properties
index 2ce2e1f..a8bb453 100644
--- a/modules/yardstick/config/benchmark-sql-dml.properties
+++ b/modules/yardstick/config/benchmark-sql-dml.properties
@@ -55,18 +55,36 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Jobs.
+j=10
+
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
#
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -bs 2 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds sql-merge-all-bs-2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -bs 6 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds sql-merge-all-bs-6-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -bs 10 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds sql-merge-all-bs-10-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlMergeBenchmark -sn IgniteNode -ds sql-merge-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlMergeQueryBenchmark -sn IgniteNode -ds sql-merge-query-1-backup\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlMergeIndexedValue1Benchmark -sn IgniteNode -ds sql-merge-indexed1-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlMergeIndexedValue2Benchmark -sn IgniteNode -ds sql-merge-indexed2-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlMergeIndexedValue8Benchmark -sn IgniteNode -ds sql-merge-indexed8-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 2 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds sql-merge-all-bs-2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 6 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds sql-merge-all-bs-6-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -bs 10 -dn IgniteSqlMergeAllBenchmark -sn IgniteNode -ds sql-merge-all-bs-10-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeBenchmark -sn IgniteNode -ds sql-merge-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeQueryBenchmark -sn IgniteNode -ds sql-merge-query-${b}-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue1Benchmark -sn IgniteNode -ds sql-merge-indexed1-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue2Benchmark -sn IgniteNode -ds sql-merge-indexed2-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlMergeIndexedValue8Benchmark -sn IgniteNode -ds sql-merge-indexed8-${b}-backup\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-store.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-store.properties b/modules/yardstick/config/benchmark-store.properties
index 746fa11..ea8487f 100644
--- a/modules/yardstick/config/benchmark-store.properties
+++ b/modules/yardstick/config/benchmark-store.properties
@@ -26,7 +26,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
--Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -50,7 +49,7 @@ BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
# Packages where the specified benchmark is searched by reflection mechanism.
BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
-# Restart servers for each benchmark.
+# Flag which indicates to restart the servers before every benchmark execution.
RESTART_SERVERS=true
# Probe point writer class name.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-tx-win.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-tx-win.properties b/modules/yardstick/config/benchmark-tx-win.properties
index 54a40b1..490b9d7 100644
--- a/modules/yardstick/config/benchmark-tx-win.properties
+++ b/modules/yardstick/config/benchmark-tx-win.properties
@@ -51,7 +51,7 @@ set DRIVER_HOSTS=localhost
:: set REMOTE_USER=
:: Run configuration which contains all benchmarks.
-:: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+:: Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
set CONFIGS=^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxBenchmark -sn IgniteNode -ds tx-put-1-backup,^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds tx-getAndPut-1-backup,^
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-tx.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-tx.properties b/modules/yardstick/config/benchmark-tx.properties
index 0d5bb02..4744dee 100644
--- a/modules/yardstick/config/benchmark-tx.properties
+++ b/modules/yardstick/config/benchmark-tx.properties
@@ -55,15 +55,30 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxBenchmark -sn IgniteNode -ds tx-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds tx-getAndPut-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteInvokeTxBenchmark -sn IgniteNode -ds tx-invoke-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds tx-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds tx-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds tx-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetTxOffHeapBenchmark -sn IgniteNode -ds tx-put-get-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetTxOffHeapValuesBenchmark -sn IgniteNode -ds tx-put-get-offheap-val-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxBenchmark -sn IgniteNode -ds tx-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds tx-getAndPut-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteInvokeTxBenchmark -sn IgniteNode -ds tx-invoke-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds tx-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds tx-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds tx-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetTxOffHeapBenchmark -sn IgniteNode -ds tx-put-get-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetTxOffHeapValuesBenchmark -sn IgniteNode -ds tx-put-get-offheap-val-${b}-backup\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark-win.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-win.properties b/modules/yardstick/config/benchmark-win.properties
index b75b5d6..e97d036 100644
--- a/modules/yardstick/config/benchmark-win.properties
+++ b/modules/yardstick/config/benchmark-win.properties
@@ -54,7 +54,7 @@ set DRIVER_HOSTS=localhost
:: set REMOTE_USER=
:: Run configuration which contains all benchmarks.
-:: Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+:: Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
set CONFIGS=^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds atomic-put-1-backup,^
-cfg %SCRIPT_DIR%\..\config\ignite-localhost-config.xml -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds atomic-put-get-1-backup,^
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/benchmark.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark.properties b/modules/yardstick/config/benchmark.properties
index cfc1499..1b5d5fe 100644
--- a/modules/yardstick/config/benchmark.properties
+++ b/modules/yardstick/config/benchmark.properties
@@ -24,7 +24,6 @@ JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false"
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" \
- -Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-verbose:gc \
-XX:+UseParNewGC \
@@ -48,6 +47,7 @@ BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe
# Packages where the specified benchmark is searched by reflection mechanism.
BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick
+# Flag which indicates to restart the servers before every benchmark execution.
RESTART_SERVERS=true
# Probe point writer class name.
@@ -65,34 +65,52 @@ DRIVER_HOSTS=localhost
# Number of nodes, used to wait for the specified number of nodes to start.
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
+# Backups count.
+b=1
+
+# Warmup.
+w=60
+
+# Duration.
+d=300
+
+# Threads count.
+t=64
+
+# Sync mode.
+sm=PRIMARY_SYNC
+
+# Jobs.
+j=10
+
# Run configuration which contains all benchmarks.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds ${ver}tx-getAndPut-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteInvokeTxBenchmark -sn IgniteNode -ds ${ver}tx-invoke-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -j 10 -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-1-backup\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -sm PRIMARY_SYNC -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -bs 1000 -sm PRIMARY_SYNC -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -bs 1000 -col -sm PRIMARY_SYNC -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-collocated-putAll-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -bs 1000 -sm PRIMARY_SYNC -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-tx-1-backup,\
--cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b 1 -w 60 -d 300 -t 64 -bs 1000 -col -sm PRIMARY_SYNC -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}atomic-collocated-putAll-tx-1-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutBenchmark -sn IgniteNode -ds ${ver}atomic-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxBenchmark -sn IgniteNode -ds ${ver}tx-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteGetAndPutTxBenchmark -sn IgniteNode -ds ${ver}tx-getAndPut-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteInvokeTxBenchmark -sn IgniteNode -ds ${ver}tx-invoke-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetTxBenchmark -sn IgniteNode -ds ${ver}tx-put-get-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryBenchmark -sn IgniteNode -ds ${ver}sql-query-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinBenchmark -sn IgniteNode -ds ${ver}sql-query-join-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutBenchmark -sn IgniteNode -ds ${ver}sql-query-put-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteAffinityCallBenchmark -sn IgniteNode -ds ${ver}affcall-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteApplyBenchmark -sn IgniteNode -ds ${ver}apply-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteBroadcastBenchmark -sn IgniteNode -ds ${ver}broad-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteExecuteBenchmark -sn IgniteNode -ds ${ver}exec-compute-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -j ${j} -dn IgniteRunBenchmark -sn IgniteNode -ds ${ver}run-compute-${b}-backup\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutGetOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-get-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}atomic-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgnitePutTxOffHeapValuesBenchmark -sn IgniteNode -ds ${ver}tx-put-offheap-val-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryJoinOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-join-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSqlQueryPutOffHeapBenchmark -sn IgniteNode -ds ${ver}sql-query-put-offheap-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -sm PRIMARY_SYNC -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -col -sm PRIMARY_SYNC -dn IgnitePutAllBenchmark -sn IgniteNode -ds ${ver}atomic-collocated-putAll-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -sm PRIMARY_SYNC -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}atomic-putAll-tx-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -bs 1000 -col -sm PRIMARY_SYNC -dn IgnitePutAllTxBenchmark -sn IgniteNode -ds ${ver}atomic-collocated-putAll-tx-${b}-backup\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/ignite-remote-config.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/ignite-remote-config.xml b/modules/yardstick/config/ignite-remote-config.xml
new file mode 100644
index 0000000..0962e53
--- /dev/null
+++ b/modules/yardstick/config/ignite-remote-config.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Ignite Spring configuration file to startup grid.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+ <import resource="ignite-base-config.xml"/>
+
+ <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration" parent="base-ignite.cfg">
+
+ <property name="discoverySpi">
+ <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
+ <property name="ipFinder">
+ <bean
+ class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
+ <property name="addresses">
+ <list>
+ <!-- In distributed environment, replace with actual hosts IP addresses. -->
+ <value>127.0.0.1:47500..47509</value>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
+ </property>
+ </bean>
+</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/test-max-int-values-offheap.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/test-max-int-values-offheap.properties b/modules/yardstick/config/test-max-int-values-offheap.properties
index 5b4da20..42769ec 100644
--- a/modules/yardstick/config/test-max-int-values-offheap.properties
+++ b/modules/yardstick/config/test-max-int-values-offheap.properties
@@ -22,7 +22,6 @@
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
- -Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-XX:-PrintGCTimeStamps \
-verbose:gc \
@@ -62,7 +61,7 @@ DRIVER_HOSTS=fosters-218
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
-cfg ${SCRIPT_DIR}/../config/ignite-int-max-values-offheap-config.xml -nn ${nodesNum} -b 0 -w 0 -d 9999999 -t 1 -sm PRIMARY_SYNC -dn IntMaxValueEntriesTest -sn IgniteNode -ds int-max-values-offheap\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/test-max-int-values-onheap.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/test-max-int-values-onheap.properties b/modules/yardstick/config/test-max-int-values-onheap.properties
index d29800a..0faf5f3 100644
--- a/modules/yardstick/config/test-max-int-values-onheap.properties
+++ b/modules/yardstick/config/test-max-int-values-onheap.properties
@@ -22,7 +22,6 @@
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
- -Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-XX:-PrintGCTimeStamps \
-verbose:gc \
@@ -62,7 +61,7 @@ DRIVER_HOSTS=fosters-218
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
-cfg ${SCRIPT_DIR}/../config/ignite-int-max-values-onheap-config.xml -nn ${nodesNum} -b 0 -w 0 -d 9999999 -t 1 -sm PRIMARY_SYNC -dn IntMaxValueEntriesTest -sn IgniteNode -ds int-max-values-onheap\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/config/test-max-int-values-swap.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/test-max-int-values-swap.properties b/modules/yardstick/config/test-max-int-values-swap.properties
index 203f004..900e8ac 100644
--- a/modules/yardstick/config/test-max-int-values-swap.properties
+++ b/modules/yardstick/config/test-max-int-values-swap.properties
@@ -22,7 +22,6 @@
# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses.
JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false \
- -Xloggc:./gc${now0}.log \
-XX:+PrintGCDetails \
-XX:-PrintGCTimeStamps \
-verbose:gc \
@@ -62,7 +61,7 @@ DRIVER_HOSTS=fosters-218
nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`))
# Run configuration.
-# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute).
+# Note that each benchmark is set to run for 300 seconds (5 min) with warm-up set to 60 seconds (1 minute).
CONFIGS="\
-cfg ${SCRIPT_DIR}/../config/ignite-int-max-values-swap-config.xml -nn ${nodesNum} -b 0 -w 0 -d 9999999 -t 1 -sm PRIMARY_SYNC -dn IntMaxValueEntriesTest -sn IgniteNode -ds int-max-values-swap\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/pom-standalone.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom-standalone.xml b/modules/yardstick/pom-standalone.xml
index 0bd5a0f..d6d5a20 100644
--- a/modules/yardstick/pom-standalone.xml
+++ b/modules/yardstick/pom-standalone.xml
@@ -29,7 +29,7 @@
<url>http://ignite.apache.org</url>
<properties>
- <yardstick.version>0.8.0</yardstick.version>
+ <yardstick.version>0.8.3</yardstick.version>
<spring.version>4.1.0.RELEASE</spring.version>
</properties>
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/modules/yardstick/pom.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index d96fcc6..f65cdbd 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -35,7 +35,7 @@
<url>http://ignite.apache.org</url>
<properties>
- <yardstick.version>0.8.2</yardstick.version>
+ <yardstick.version>0.8.3</yardstick.version>
<spring.version>4.1.0.RELEASE</spring.version>
</properties>
http://git-wip-us.apache.org/repos/asf/ignite/blob/1a08ef76/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 238361b..9d86a24 100644
--- a/pom.xml
+++ b/pom.xml
@@ -507,10 +507,13 @@
<fileset dir="${basedir}/target/release-package/benchmarks/config/">
<include name="*.*" />
<exclude name="benchmark.properties"/>
+ <exclude name="benchmark-remote.properties"/>
<exclude name="benchmark-sample.properties"/>
+ <exclude name="benchmark-remote-sample.properties"/>
<exclude name="benchmark-multicast.properties"/>
<exclude name="ignite-base-config.xml"/>
<exclude name="ignite-localhost-config.xml"/>
+ <exclude name="ignite-remote-config.xml"/>
<exclude name="ignite-multicast-config.xml"/>
</fileset>
</delete>
@@ -542,7 +545,7 @@
tofile="${basedir}/target/release-package/benchmarks/README.txt" overwrite="true">
</copy>
- <copy file="${basedir}/modules/yardstick/DEVNOTES.txt"
+ <copy file="${basedir}/modules/yardstick/DEVNOTES-standalone.txt"
tofile="${basedir}/target/release-package/benchmarks/sources/DEVNOTES.txt"/>
</target>
</configuration>
[06/50] [abbrv] ignite git commit: IGNITE-4409: Fixed UUID
deserialization issue during DML command execution. This closes #1554.
Posted by av...@apache.org.
IGNITE-4409: Fixed UUID deserialization issue during DML command execution. This closes #1554.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/32c3d6bd
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/32c3d6bd
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/32c3d6bd
Branch: refs/heads/master
Commit: 32c3d6bd66030b70ede6d1dfa58fd81eee7829ba
Parents: 0130b09
Author: Alexander Paschenko <al...@gmail.com>
Authored: Mon Feb 20 18:04:20 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Mon Feb 20 18:04:20 2017 +0300
----------------------------------------------------------------------
.../query/h2/DmlStatementsProcessor.java | 48 ++++++++++++++------
...niteCacheAbstractInsertSqlQuerySelfTest.java | 6 ++-
.../IgniteCacheInsertSqlQuerySelfTest.java | 14 ++++++
3 files changed, 54 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/32c3d6bd/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index 78c5bbc..60bc483 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -55,6 +55,7 @@ import org.apache.ignite.internal.processors.query.GridQueryCancel;
import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
import org.apache.ignite.internal.processors.query.GridQueryFieldsResultAdapter;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
@@ -577,8 +578,11 @@ public class DmlStatementsProcessor {
if (hasNewVal && i == valColIdx - 2)
continue;
- newColVals.put(plan.colNames[i], convert(e.get(i + 2), plan.colNames[i],
- plan.tbl.rowDescriptor(), plan.colTypes[i]));
+ GridQueryProperty prop = plan.tbl.rowDescriptor().type().property(plan.colNames[i]);
+
+ assert prop != null;
+
+ newColVals.put(plan.colNames[i], convert(e.get(i + 2), desc, prop.type(), plan.colTypes[i]));
}
newVal = plan.valSupplier.apply(e);
@@ -668,24 +672,18 @@ public class DmlStatementsProcessor {
* Convert value to column's expected type by means of H2.
*
* @param val Source value.
- * @param colName Column name to search for property.
* @param desc Row descriptor.
+ * @param expCls Expected value class.
* @param type Expected column type to convert to.
* @return Converted object.
* @throws IgniteCheckedException if failed.
*/
@SuppressWarnings({"ConstantConditions", "SuspiciousSystemArraycopy"})
- private static Object convert(Object val, String colName, GridH2RowDescriptor desc, int type)
+ private static Object convert(Object val, GridH2RowDescriptor desc, Class<?> expCls, int type)
throws IgniteCheckedException {
if (val == null)
return null;
- GridQueryProperty prop = desc.type().property(colName);
-
- assert prop != null;
-
- Class<?> expCls = prop.type();
-
Class<?> currCls = val.getClass();
if (val instanceof Date && currCls != Date.class && expCls == Date.class) {
@@ -694,6 +692,11 @@ public class DmlStatementsProcessor {
return new Date(((Date) val).getTime());
}
+ // User-given UUID is always serialized by H2 to byte array, so we have to deserialize manually
+ if (type == Value.UUID && currCls == byte[].class)
+ return U.unmarshal(desc.context().marshaller(), (byte[]) val,
+ U.resolveClassLoader(desc.context().gridConfig()));
+
// We have to convert arrays of reference types manually - see https://issues.apache.org/jira/browse/IGNITE-4327
// Still, we only can convert from Object[] to something more precise.
if (type == Value.ARRAY && currCls != expCls) {
@@ -935,22 +938,41 @@ public class DmlStatementsProcessor {
private IgniteBiTuple<?, ?> rowToKeyValue(GridCacheContext cctx, Object[] row, String[] cols,
int[] colTypes, KeyValueSupplier keySupplier, KeyValueSupplier valSupplier, int keyColIdx, int valColIdx,
GridH2RowDescriptor rowDesc) throws IgniteCheckedException {
+ GridQueryTypeDescriptor desc = rowDesc.type();
+
Object key = keySupplier.apply(F.asList(row));
+
+ if (GridQueryProcessor.isSqlType(desc.keyClass())) {
+ assert keyColIdx != -1;
+
+ key = convert(key, rowDesc, desc.keyClass(), colTypes[keyColIdx]);
+ }
+
Object val = valSupplier.apply(F.asList(row));
+ if (GridQueryProcessor.isSqlType(desc.valueClass())) {
+ assert valColIdx != -1;
+
+ val = convert(val, rowDesc, desc.valueClass(), colTypes[valColIdx]);
+ }
+
if (key == null)
throw new IgniteSQLException("Key for INSERT or MERGE must not be null", IgniteQueryErrorCode.NULL_KEY);
if (val == null)
throw new IgniteSQLException("Value for INSERT or MERGE must not be null", IgniteQueryErrorCode.NULL_VALUE);
- GridQueryTypeDescriptor desc = rowDesc.type();
-
for (int i = 0; i < cols.length; i++) {
if (i == keyColIdx || i == valColIdx)
continue;
- desc.setValue(cols[i], key, val, convert(row[i], cols[i], rowDesc, colTypes[i]));
+ GridQueryProperty prop = desc.property(cols[i]);
+
+ assert prop != null;
+
+ Class<?> expCls = prop.type();
+
+ desc.setValue(cols[i], key, val, convert(row[i], rowDesc, expCls, colTypes[i]));
}
if (cctx.binaryMarshaller()) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/32c3d6bd/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
index 86d01c7..4470a48 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractInsertSqlQuerySelfTest.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.binary.BinaryAbstractIdentityResolver;
import org.apache.ignite.binary.BinaryArrayIdentityResolver;
@@ -131,18 +132,21 @@ public abstract class IgniteCacheAbstractInsertSqlQuerySelfTest extends GridComm
/**
*
*/
- protected void createCaches() {
+ void createCaches() {
ignite(0).createCache(cacheConfig("S2P", true, false, String.class, Person.class, String.class, String.class));
ignite(0).createCache(cacheConfig("I2P", true, false, Integer.class, Person.class));
ignite(0).createCache(cacheConfig("K2P", true, false, Key.class, Person.class));
ignite(0).createCache(cacheConfig("K22P", true, true, Key2.class, Person2.class));
ignite(0).createCache(cacheConfig("I2I", true, false, Integer.class, Integer.class));
+ ignite(0).createCache(cacheConfig("U2I", true, false, UUID.class, Integer.class));
}
/**
*
*/
final void createBinaryCaches() {
+ ignite(0).createCache(cacheConfig("U2I", true, false, UUID.class, Integer.class));
+
{
CacheConfiguration s2pCcfg = cacheConfig("S2P", true, false);
http://git-wip-us.apache.org/repos/asf/ignite/blob/32c3d6bd/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
index e9c21dc..471b791 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInsertSqlQuerySelfTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.cache;
+import java.util.UUID;
import java.util.concurrent.Callable;
import javax.cache.CacheException;
import org.apache.ignite.IgniteCache;
@@ -202,4 +203,17 @@ public class IgniteCacheInsertSqlQuerySelfTest extends IgniteCacheAbstractInsert
assertEquals(createPerson(2, "Alex"), p.get(new Key4(2)));
}
+
+ /**
+ *
+ */
+ public void testUuidHandling() {
+ IgniteCache<UUID, Integer> p = ignite(0).cache("U2I");
+
+ UUID id = UUID.randomUUID();
+
+ p.query(new SqlFieldsQuery("insert into Integer(_key, _val) values (?, ?)").setArgs(id, 1));
+
+ assertEquals(1, (int)p.get(id));
+ }
}
[29/50] [abbrv] ignite git commit: Review minors.
Posted by av...@apache.org.
Review minors.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9bc776fe
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9bc776fe
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9bc776fe
Branch: refs/heads/master
Commit: 9bc776fef537d2fb749951cbbf186e81858c11b5
Parents: de6e52b
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 19:25:14 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 19:29:59 2017 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheProcessor.java | 2 +-
.../query/h2/opt/GridH2SpatialIndex.java | 23 +++-------
.../h2/GridH2IndexingSegmentedGeoSelfTest.java | 2 +
.../query/h2/opt/GridH2IndexBase.java | 38 ++++++++++++++++-
.../query/h2/opt/GridH2TreeIndex.java | 44 +-------------------
.../query/IgniteSqlSplitterSelfTest.java | 2 +-
6 files changed, 46 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bc776fe/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index c3e3f3b..50e1379 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -270,7 +270,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
cfg.setBackups(Integer.MAX_VALUE);
if( cfg.getQueryParallelism() > 1 && cfg.getCacheMode() != PARTITIONED)
- throw new IgniteCheckedException("Cache index segmentation is supported for PARTITIONED mode only.");
+ throw new IgniteCheckedException("Segmented indices are supported for PARTITIONED mode only.");
if (cfg.getAffinityMapper() == null)
cfg.setAffinityMapper(cacheObjCtx.defaultAffMapper());
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bc776fe/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
index 096b82d..c3a1362 100644
--- a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
+++ b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
@@ -121,6 +121,8 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
for (int i = 0; i < segmentsCnt; i++)
segments[i] = store.openMap("spatialIndex-" + i, new MVRTreeMap.Builder<Long>());
+
+ ctx = tbl.rowDescriptor().context();
}
/**
@@ -156,13 +158,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
assert key != null;
- Long rowId = keyToId.get(key);
+ final int seg = segmentForRow(row);
- int seg;
+ Long rowId = keyToId.get(key);
if (rowId != null) {
- seg = segmentForRowID(rowId);
-
Long oldRowId = segments[seg].remove(getEnvelope(idToRow.get(rowId), rowId));
assert rowId.equals(oldRowId);
@@ -170,8 +170,6 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
else {
rowId = ++rowIds;
- seg = segmentForRowID(rowId);
-
keyToId.put(key, rowId);
}
@@ -190,17 +188,6 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
}
/**
- * @param id Row ID.
- *
- * @return Segment ID for given row ID.
- */
- private int segmentForRowID(Long id) {
- assert id != null;
-
- return (int)(id % segmentsCount());
- }
-
- /**
* @param row Row.
* @param rowId Row id.
* @return Envelope.
@@ -235,7 +222,7 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
assert oldRow != null;
- int seg = segmentForRowID(rowId);
+ final int seg = segmentForRow(row);
if (!segments[seg].remove(getEnvelope(row, rowId), rowId))
throw DbException.throwInternalError("row not found");
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bc776fe/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
index b806321..e404f38 100644
--- a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
+++ b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
@@ -27,4 +27,6 @@ public class GridH2IndexingSegmentedGeoSelfTest extends GridH2IndexingGeoSelfTes
@Override protected CacheConfiguration cacheConfiguration(String gridName) throws Exception {
return super.cacheConfiguration(gridName).setQueryParallelism(7);
}
+
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bc776fe/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 31057c7..fc5eb4b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -43,6 +43,7 @@ import org.apache.ignite.internal.GridTopic;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2IndexRangeRequest;
@@ -126,6 +127,8 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
};
+ protected GridCacheContext<?, ?> ctx;
+
/**
* @param tbl Table.
*/
@@ -133,6 +136,8 @@ public abstract class GridH2IndexBase extends BaseIndex {
final GridH2RowDescriptor desc = tbl.rowDescriptor();
if (desc != null && desc.context() != null) {
+ ctx = desc.context();
+
GridKernalContext ctx = desc.context().kernalContext();
log = ctx.log(getClass());
@@ -183,11 +188,11 @@ public abstract class GridH2IndexBase extends BaseIndex {
* @return Index segment ID for current query context.
*/
protected int threadLocalSegment() {
- GridH2QueryContext qctx = GridH2QueryContext.get();
-
if(segmentsCount() == 1)
return 0;
+ GridH2QueryContext qctx = GridH2QueryContext.get();
+
if(qctx == null)
throw new IllegalStateException("GridH2QueryContext is not initialized.");
@@ -864,6 +869,35 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
/**
+ * @param row Table row.
+ * @return Segment ID for given row.
+ */
+ protected int segmentForRow(SearchRow row) {
+ assert row != null;
+
+ CacheObject key;
+
+ if (ctx != null) {
+ final Value keyColValue = row.getValue(KEY_COL);
+
+ assert keyColValue != null;
+
+ final Object o = keyColValue.getObject();
+
+ if (o instanceof CacheObject)
+ key = (CacheObject)o;
+ else
+ key = ctx.toCacheKeyObject(o);
+
+ return segmentForPartition(ctx.affinity().partition(key));
+ }
+
+ assert segmentsCount() == 1;
+
+ return 0;
+ }
+
+ /**
* Simple cursor from a single node.
*/
private static class UnicastCursor implements Cursor {
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bc776fe/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 80597f2..2873211 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -17,15 +17,12 @@
package org.apache.ignite.internal.processors.query.h2.opt;
-import java.lang.reflect.Field;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.util.GridEmptyIterator;
import org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap;
import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeGuard;
@@ -45,8 +42,6 @@ import org.h2.table.TableFilter;
import org.h2.value.Value;
import org.jetbrains.annotations.Nullable;
-import static org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.KEY_COL;
-
/**
* Base class for snapshotable segmented tree indexes.
*/
@@ -58,9 +53,6 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
/** */
private final boolean snapshotEnabled;
- /** */
- private final GridH2RowDescriptor desc;
-
/**
* Constructor with index initialization. Creates index with single segment.
*
@@ -91,8 +83,6 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
IndexColumn.mapColumns(cols, tbl);
- desc = tbl.rowDescriptor();
-
initBaseIndex(tbl, 0, name, cols,
pk ? IndexType.createPrimaryKey(false, false) : IndexType.createNonUnique(false, false, false));
@@ -106,8 +96,7 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
if (snapshotEnabled) {
for (int i = 0; i < segmentsCnt; i++) {
segments[i] = new SnapTreeMap<GridSearchRowPointer, GridH2Row>(this) {
- @Override
- protected void afterNodeUpdate_nl(Node<GridSearchRowPointer, GridH2Row> node, Object val) {
+ @Override protected void afterNodeUpdate_nl(Node<GridSearchRowPointer, GridH2Row> node, Object val) {
if (val != null)
node.key = (GridSearchRowPointer)val;
}
@@ -426,37 +415,6 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
}
/**
- * @param row Table row.
- * @return Segment ID for given row.
- */
- private int segmentForRow(SearchRow row) {
- assert row != null;
-
- CacheObject key;
-
- if (desc != null && desc.context() != null) {
- GridCacheContext<?, ?> ctx = desc.context();
-
- assert ctx != null;
-
- final Value keyColValue = row.getValue(KEY_COL);
-
- assert keyColValue != null;
-
- final Object o = keyColValue.getObject();
-
- if (o instanceof CacheObject)
- key = (CacheObject)o;
- else
- key = ctx.toCacheKeyObject(o);
-
- return segmentForPartition(ctx.affinity().partition(key));
- }
- else
- return 0;
- }
-
- /**
* Comparable row with bias. Will be used for queries to have correct bounds (in case of multicolumn index
* and query on few first columns we will multiple equal entries in tree).
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/9bc776fe/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 37dea47..69f66a5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -824,7 +824,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
return null;
}
- }, CacheException.class, "Cache index segmentation is supported for PARTITIONED mode only.");
+ }, CacheException.class, " Segmented indices are supported for PARTITIONED mode only.");
}
/**
[31/50] [abbrv] ignite git commit: Distributed joins with segmented
spatial index tests added.
Posted by av...@apache.org.
Distributed joins with segmented spatial index tests added.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4dc36948
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4dc36948
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4dc36948
Branch: refs/heads/master
Commit: 4dc369487ef6f5583348f6a0232952c441623723
Parents: 9bc776f
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 21:13:13 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 21:13:13 2017 +0300
----------------------------------------------------------------------
.../query/h2/GridH2IndexingGeoSelfTest.java | 405 ++++++++++++++-----
.../h2/GridH2IndexingSegmentedGeoSelfTest.java | 13 +-
2 files changed, 302 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4dc36948/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
index 2843076..839514b 100644
--- a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
+++ b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.query.h2;
import com.vividsolutions.jts.geom.Geometry;
+import com.vividsolutions.jts.io.ParseException;
import com.vividsolutions.jts.io.WKTReader;
import java.io.Serializable;
import java.util.Arrays;
@@ -31,14 +32,18 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import javax.cache.Cache;
import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.GridTestUtils;
+import org.jetbrains.annotations.NotNull;
/**
* Geo-indexing test.
@@ -50,6 +55,12 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
/** */
private static final long DUR = 60000L;
+ /** Number of generated samples. */
+ public static final int ENEMYCAMP_SAMPLES_COUNT = 500;
+
+ /** Number of generated samples. */
+ public static final int ENEMY_SAMPLES_COUNT = 1000;
+
/** {@inheritDoc} */
@Override protected int gridCount() {
return 3;
@@ -60,29 +71,41 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
return DUR * 3;
}
- /** {@inheritDoc} */
- @Override protected Class<?>[] indexedTypes() {
- return new Class<?>[]{
- Integer.class, EnemyCamp.class,
- Long.class, Geometry.class // Geometry must be indexed here.
- };
+ /**
+ * @param name Cache name.
+ * @param partitioned Partition or replicated cache.
+ * @param idxTypes Indexed types.
+ * @return Cache configuration.
+ */
+ protected <K, V> CacheConfiguration<K, V> cacheConfig(String name, boolean partitioned,
+ Class<?>... idxTypes) throws Exception {
+ return new CacheConfiguration<K, V>(name)
+ .setName(name)
+ .setCacheMode(partitioned ? CacheMode.PARTITIONED : CacheMode.REPLICATED)
+ .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+ .setIndexedTypes(idxTypes);
}
/**
* @throws Exception If failed.
*/
public void testPrimitiveGeometry() throws Exception {
- IgniteCache<Long, Geometry> cache = grid(0).cache(null);
+ IgniteCache<Long, Geometry> cache = grid(0).getOrCreateCache(cacheConfig("geom", true, Long.class, Geometry.class));
- WKTReader r = new WKTReader();
+ try {
+ WKTReader r = new WKTReader();
- for (long i = 0; i < 100; i++)
- cache.put(i, r.read("POINT(" + i + " " + i + ")"));
+ for (long i = 0; i < 100; i++)
+ cache.put(i, r.read("POINT(" + i + " " + i + ")"));
- List<List<?>> res = cache.query(new SqlFieldsQuery("explain select _key from Geometry where _val && ?")
- .setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))")).setLocal(true)).getAll();
+ List<List<?>> res = cache.query(new SqlFieldsQuery("explain select _key from Geometry where _val && ?")
+ .setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))")).setLocal(true)).getAll();
- assertTrue("__ explain: " + res, res.get(0).get(0).toString().contains("_val_idx"));
+ assertTrue("__ explain: " + res, res.get(0).get(0).toString().contains("_val_idx"));
+ }
+ finally {
+ cache.destroy();
+ }
}
/**
@@ -90,59 +113,65 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
*/
@SuppressWarnings("unchecked")
public void testGeo() throws Exception {
- IgniteCache<Integer, EnemyCamp> cache = grid(0).cache(null);
+ IgniteCache<Integer, EnemyCamp> cache = grid(0).getOrCreateCache(
+ cacheConfig("camp", true, Integer.class, EnemyCamp.class));
- WKTReader r = new WKTReader();
+ try {
+ WKTReader r = new WKTReader();
- cache.getAndPut(0, new EnemyCamp(r.read("POINT(25 75)"), "A"));
- cache.getAndPut(1, new EnemyCamp(r.read("POINT(70 70)"), "B"));
- cache.getAndPut(2, new EnemyCamp(r.read("POINT(70 30)"), "C"));
- cache.getAndPut(3, new EnemyCamp(r.read("POINT(75 25)"), "D"));
+ cache.getAndPut(0, new EnemyCamp(r.read("POINT(25 75)"), "A"));
+ cache.getAndPut(1, new EnemyCamp(r.read("POINT(70 70)"), "B"));
+ cache.getAndPut(2, new EnemyCamp(r.read("POINT(70 30)"), "C"));
+ cache.getAndPut(3, new EnemyCamp(r.read("POINT(75 25)"), "D"));
- SqlQuery<Integer, EnemyCamp> qry = new SqlQuery(EnemyCamp.class, "coords && ?");
+ SqlQuery<Integer, EnemyCamp> qry = new SqlQuery(EnemyCamp.class, "coords && ?");
- Collection<Cache.Entry<Integer, EnemyCamp>> res = cache.query(
- qry.setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))"))).getAll();
+ Collection<Cache.Entry<Integer, EnemyCamp>> res = cache.query(
+ qry.setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))"))).getAll();
- checkPoints(res, "A");
+ checkPoints(res, "A");
- res = cache.query(
- qry.setArgs(r.read("POLYGON((10 5, 10 35, 70 30, 75 25, 10 5))"))).getAll();
+ res = cache.query(
+ qry.setArgs(r.read("POLYGON((10 5, 10 35, 70 30, 75 25, 10 5))"))).getAll();
- checkPoints(res, "C", "D");
+ checkPoints(res, "C", "D");
- // Move B to the first polygon.
- cache.getAndPut(1, new EnemyCamp(r.read("POINT(20 75)"), "B"));
+ // Move B to the first polygon.
+ cache.getAndPut(1, new EnemyCamp(r.read("POINT(20 75)"), "B"));
- res = cache.query(
- qry.setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))"))).getAll();
+ res = cache.query(
+ qry.setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))"))).getAll();
- checkPoints(res, "A", "B");
+ checkPoints(res, "A", "B");
- // Move B to the second polygon.
- cache.getAndPut(1, new EnemyCamp(r.read("POINT(30 30)"), "B"));
+ // Move B to the second polygon.
+ cache.getAndPut(1, new EnemyCamp(r.read("POINT(30 30)"), "B"));
- res = cache.query(
- qry.setArgs(r.read("POLYGON((10 5, 10 35, 70 30, 75 25, 10 5))"))).getAll();
+ res = cache.query(
+ qry.setArgs(r.read("POLYGON((10 5, 10 35, 70 30, 75 25, 10 5))"))).getAll();
- checkPoints(res, "B", "C", "D");
+ checkPoints(res, "B", "C", "D");
- // Remove B.
- cache.getAndRemove(1);
+ // Remove B.
+ cache.getAndRemove(1);
- res = cache.query(
- qry.setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))"))).getAll();
+ res = cache.query(
+ qry.setArgs(r.read("POLYGON((5 70, 5 80, 30 80, 30 70, 5 70))"))).getAll();
- checkPoints(res, "A");
+ checkPoints(res, "A");
- res = cache.query(
- qry.setArgs(r.read("POLYGON((10 5, 10 35, 70 30, 75 25, 10 5))"))).getAll();
+ res = cache.query(
+ qry.setArgs(r.read("POLYGON((10 5, 10 35, 70 30, 75 25, 10 5))"))).getAll();
- checkPoints(res, "C", "D");
+ checkPoints(res, "C", "D");
- // Check explaint request.
- assertTrue(F.first(cache.query(new SqlFieldsQuery("explain select * from EnemyCamp " +
- "where coords && 'POINT(25 75)'")).getAll()).get(0).toString().contains("coords_idx"));
+ // Check explaint request.
+ assertTrue(F.first(cache.query(new SqlFieldsQuery("explain select * from EnemyCamp " +
+ "where coords && 'POINT(25 75)'")).getAll()).get(0).toString().contains("coords_idx"));
+ }
+ finally {
+ cache.destroy();
+ }
}
/**
@@ -150,100 +179,107 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
*/
@SuppressWarnings("unchecked")
public void testGeoMultithreaded() throws Exception {
- final IgniteCache<Integer, EnemyCamp> cache1 = grid(0).cache(null);
- final IgniteCache<Integer, EnemyCamp> cache2 = grid(1).cache(null);
- final IgniteCache<Integer, EnemyCamp> cache3 = grid(2).cache(null);
+ final CacheConfiguration<Integer, EnemyCamp> ccfg = cacheConfig("camp", true, Integer.class, EnemyCamp.class);
- final String[] points = new String[CNT];
+ final IgniteCache<Integer, EnemyCamp> cache1 = grid(0).getOrCreateCache(ccfg);
+ final IgniteCache<Integer, EnemyCamp> cache2 = grid(1).cache("camp");
+ final IgniteCache<Integer, EnemyCamp> cache3 = grid(2).cache("camp");
- WKTReader r = new WKTReader();
+ try {
+ final String[] points = new String[CNT];
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
+ WKTReader r = new WKTReader();
- for (int idx = 0; idx < CNT; idx++) {
- int x = rnd.nextInt(1, 100);
- int y = rnd.nextInt(1, 100);
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
- cache1.getAndPut(idx, new EnemyCamp(r.read("POINT(" + x + " " + y + ")"), Integer.toString(idx)));
+ for (int idx = 0; idx < CNT; idx++) {
+ int x = rnd.nextInt(1, 100);
+ int y = rnd.nextInt(1, 100);
- points[idx] = Integer.toString(idx);
- }
+ cache1.getAndPut(idx, new EnemyCamp(r.read("POINT(" + x + " " + y + ")"), Integer.toString(idx)));
- Thread.sleep(200);
+ points[idx] = Integer.toString(idx);
+ }
- final AtomicBoolean stop = new AtomicBoolean();
- final AtomicReference<Exception> err = new AtomicReference<>();
+ Thread.sleep(200);
- IgniteInternalFuture<?> putFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- WKTReader r = new WKTReader();
+ final AtomicBoolean stop = new AtomicBoolean();
+ final AtomicReference<Exception> err = new AtomicReference<>();
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
+ IgniteInternalFuture<?> putFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ WKTReader r = new WKTReader();
- while (!stop.get()) {
- int cacheIdx = rnd.nextInt(0, 3);
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
- IgniteCache<Integer, EnemyCamp> cache = cacheIdx == 0 ? cache1 : cacheIdx == 1 ? cache2 : cache3;
+ while (!stop.get()) {
+ int cacheIdx = rnd.nextInt(0, 3);
- int idx = rnd.nextInt(CNT);
- int x = rnd.nextInt(1, 100);
- int y = rnd.nextInt(1, 100);
+ IgniteCache<Integer, EnemyCamp> cache = cacheIdx == 0 ? cache1 : cacheIdx == 1 ? cache2 : cache3;
- cache.getAndPut(idx, new EnemyCamp(r.read("POINT(" + x + " " + y + ")"), Integer.toString(idx)));
+ int idx = rnd.nextInt(CNT);
+ int x = rnd.nextInt(1, 100);
+ int y = rnd.nextInt(1, 100);
- U.sleep(50);
- }
+ cache.getAndPut(idx, new EnemyCamp(r.read("POINT(" + x + " " + y + ")"), Integer.toString(idx)));
- return null;
- }
- }, Runtime.getRuntime().availableProcessors(), "put-thread");
+ U.sleep(50);
+ }
- IgniteInternalFuture<?> qryFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- WKTReader r = new WKTReader();
+ return null;
+ }
+ }, Runtime.getRuntime().availableProcessors(), "put-thread");
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
+ IgniteInternalFuture<?> qryFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ WKTReader r = new WKTReader();
- while (!stop.get()) {
- try {
- int cacheIdx = rnd.nextInt(0, 3);
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
- IgniteCache<Integer, EnemyCamp> cache = cacheIdx == 0 ? cache1 : cacheIdx == 1 ? cache2 : cache3;
+ while (!stop.get()) {
+ try {
+ int cacheIdx = rnd.nextInt(0, 3);
- SqlQuery<Integer, EnemyCamp> qry = new SqlQuery<>(
- EnemyCamp.class, "coords && ?");
+ IgniteCache<Integer, EnemyCamp> cache = cacheIdx == 0 ? cache1 : cacheIdx == 1 ? cache2 : cache3;
- Collection<Cache.Entry<Integer, EnemyCamp>> res = cache.query(qry.setArgs(
- r.read("POLYGON((0 0, 0 100, 100 100, 100 0, 0 0))"))).getAll();
+ SqlQuery<Integer, EnemyCamp> qry = new SqlQuery<>(
+ EnemyCamp.class, "coords && ?");
- checkPoints(res, points);
+ Collection<Cache.Entry<Integer, EnemyCamp>> res = cache.query(qry.setArgs(
+ r.read("POLYGON((0 0, 0 100, 100 100, 100 0, 0 0))"))).getAll();
- U.sleep(5);
- }
- catch (Exception e) {
- err.set(e);
+ checkPoints(res, points);
- stop.set(true);
+ U.sleep(5);
+ }
+ catch (Exception e) {
+ err.set(e);
- break;
+ stop.set(true);
+
+ break;
+ }
}
- }
- return null;
- }
- }, 4, "qry-thread");
+ return null;
+ }
+ }, 4, "qry-thread");
- U.sleep(60000L);
+ U.sleep(6000L);
- stop.set(true);
+ stop.set(true);
- putFut.get();
- qryFut.get();
+ putFut.get();
+ qryFut.get();
- Exception err0 = err.get();
+ Exception err0 = err.get();
- if (err0 != null)
- throw err0;
+ if (err0 != null)
+ throw err0;
+ }
+ finally {
+ cache1.destroy();
+ }
}
/**
@@ -252,7 +288,7 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
* @param res Result.
* @param points Expected points.
*/
- private void checkPoints( Collection<Cache.Entry<Integer, EnemyCamp>> res, String... points) {
+ private void checkPoints(Collection<Cache.Entry<Integer, EnemyCamp>> res, String... points) {
Set<String> set = new HashSet<>(Arrays.asList(points));
assertEquals(set.size(), res.size());
@@ -262,12 +298,157 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
}
/**
+ * @throws Exception if fails.
+ */
+ public void testSegmentedGeoIndexJoin() throws Exception {
+ IgniteCache<Integer, Enemy> c1 = ignite(0).getOrCreateCache(cacheConfig("enemy", true, Integer.class, Enemy.class));
+ IgniteCache<Integer, EnemyCamp> c2 = ignite(0).getOrCreateCache(cacheConfig("camp", true, Integer.class, EnemyCamp.class));
+
+ try {
+ fillCache();
+
+ checkDistributedQuery();
+
+ checkLocalQuery();
+ }
+ finally {
+ c1.destroy();
+ c2.destroy();
+ }
+ }
+
+ /**
+ * @throws Exception if fails.
+ */
+ public void testSegmentedGeoIndexJoin2() throws Exception {
+ IgniteCache<Integer, Enemy> c1 = ignite(0).getOrCreateCache(cacheConfig("enemy", true, Integer.class, Enemy.class));
+ IgniteCache<Integer, EnemyCamp> c2 = ignite(0).getOrCreateCache(cacheConfig("camp", false, Integer.class, EnemyCamp.class));
+
+ try {
+ fillCache();
+
+ checkDistributedQuery();
+
+ checkLocalQuery();
+ }
+ finally {
+ c1.destroy();
+ c2.destroy();
+ }
+ }
+
+ /** */
+ private void checkDistributedQuery() throws ParseException {
+ IgniteCache<Integer, Enemy> c1 = ignite(0).cache("enemy");
+ IgniteCache<Integer, EnemyCamp> c2 = ignite(0).cache("camp");
+
+ final Geometry lethalArea = new WKTReader().read("POLYGON((30 30, 30 70, 70 70, 70 30, 30 30))");
+
+ int expectedEnemies = 0;
+
+ for (Cache.Entry<Integer, Enemy> e : c1) {
+ final Integer campID = e.getValue().campId;
+
+ if (30 <= campID && campID < ENEMYCAMP_SAMPLES_COUNT) {
+ final EnemyCamp camp = c2.get(campID);
+
+ if (lethalArea.covers(camp.coords))
+ expectedEnemies++;
+ }
+ }
+
+ final SqlFieldsQuery query = new SqlFieldsQuery("select e._val, c._val from \"enemy\".Enemy e, \"camp\".EnemyCamp c " +
+ "where e.campId = c._key and c.coords && ?").setArgs(lethalArea);
+
+ List<List<?>> result = c1.query(query.setDistributedJoins(true)).getAll();
+
+ assertEquals(expectedEnemies, result.size());
+ }
+
+ /** */
+ private void checkLocalQuery() throws ParseException {
+ IgniteCache<Integer, Enemy> c1 = ignite(0).cache("enemy");
+ IgniteCache<Integer, EnemyCamp> c2 = ignite(0).cache("camp");
+
+ final Geometry lethalArea = new WKTReader().read("POLYGON((30 30, 30 70, 70 70, 70 30, 30 30))");
+
+ Set<Integer> localCampsIDs = new HashSet<>();
+
+ for(Cache.Entry<Integer, EnemyCamp> e : c2.localEntries())
+ localCampsIDs.add(e.getKey());
+
+ int expectedEnemies = 0;
+
+ for (Cache.Entry<Integer, Enemy> e : c1.localEntries()) {
+ final Integer campID = e.getValue().campId;
+
+ if (localCampsIDs.contains(campID)) {
+ final EnemyCamp camp = c2.get(campID);
+
+ if (lethalArea.covers(camp.coords))
+ expectedEnemies++;
+ }
+ }
+
+ final SqlFieldsQuery query = new SqlFieldsQuery("select e._val, c._val from \"enemy\".Enemy e, \"camp\".EnemyCamp c " +
+ "where e.campId = c._key and c.coords && ?").setArgs(lethalArea);
+
+ List<List<?>> result = c1.query(query.setLocal(true)).getAll();
+
+ assertEquals(expectedEnemies, result.size());
+ }
+
+ /** */
+ private void fillCache() throws ParseException {
+ IgniteCache<Integer, Enemy> c1 = ignite(0).cache("enemy");
+ IgniteCache<Integer, EnemyCamp> c2 = ignite(0).cache("camp");
+
+ final ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ WKTReader r = new WKTReader();
+
+ for (int i = 0; i < ENEMYCAMP_SAMPLES_COUNT; i++) {
+ final String point = String.format("POINT(%d %d)", rnd.nextInt(100), rnd.nextInt(100));
+
+ c2.put(i, new EnemyCamp(r.read(point), "camp-" + i));
+ }
+
+ for (int i = 0; i < ENEMY_SAMPLES_COUNT; i++) {
+ int campID = 30 + rnd.nextInt(ENEMYCAMP_SAMPLES_COUNT + 10);
+
+ c1.put(i, new Enemy(campID, "enemy-" + i));
+ }
+ }
+
+ /**
+ *
+ */
+ private static class Enemy {
+ /** */
+ @QuerySqlField
+ int campId;
+
+ /** */
+ @QuerySqlField
+ String name;
+
+ /**
+ * @param campId Camp ID.
+ * @param name Name.
+ */
+ public Enemy(int campId, String name) {
+ this.campId = campId;
+ this.name = name;
+ }
+ }
+
+ /**
*
*/
- private static class EnemyCamp implements Serializable {
+ protected static class EnemyCamp implements Serializable {
/** */
@QuerySqlField(index = true)
- private Geometry coords;
+ Geometry coords;
/** */
@QuerySqlField
@@ -277,7 +458,7 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
* @param coords Coordinates.
* @param name Name.
*/
- private EnemyCamp(Geometry coords, String name) {
+ EnemyCamp(Geometry coords, String name) {
this.coords = coords;
this.name = name;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4dc36948/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
index e404f38..eb0fd0f 100644
--- a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
+++ b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
@@ -23,10 +23,15 @@ import org.apache.ignite.configuration.CacheConfiguration;
* Test for segmented geo index.
*/
public class GridH2IndexingSegmentedGeoSelfTest extends GridH2IndexingGeoSelfTest {
- /** {@inheritDoc} */
- @Override protected CacheConfiguration cacheConfiguration(String gridName) throws Exception {
- return super.cacheConfiguration(gridName).setQueryParallelism(7);
- }
+ /** */
+ private static int QRY_PARALLELISM_LVL = 7;
+ /** {@inheritDoc} */
+ @Override
+ protected <K, V> CacheConfiguration<K, V> cacheConfig(String name, boolean partitioned,
+ Class<?>... idxTypes) throws Exception {
+ final CacheConfiguration<K, V> ccfg = super.cacheConfig(name, partitioned, idxTypes);
+ return ccfg.setQueryParallelism(partitioned ? QRY_PARALLELISM_LVL : 1);
+ }
}
\ No newline at end of file
[42/50] [abbrv] ignite git commit: Merge remote-tracking branch
'origin/ignite-1.9' into ignite-1.9
Posted by av...@apache.org.
Merge remote-tracking branch 'origin/ignite-1.9' into ignite-1.9
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/80352d0a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/80352d0a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/80352d0a
Branch: refs/heads/master
Commit: 80352d0a8e0a5f69a33d15d30d5b1f014c247e61
Parents: 7be2a70 e3d7326
Author: devozerov <vo...@gridgain.com>
Authored: Wed Feb 22 16:36:13 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Feb 22 16:36:13 2017 +0300
----------------------------------------------------------------------
DEVNOTES.txt | 2 +-
examples/pom.xml | 14 ++
.../examples/java8/spark/SharedRDDExample.java | 110 ----------------
.../ignite/examples/spark/SharedRDDExample.java | 127 +++++++++++++++++++
.../ignite/examples/spark/package-info.java | 22 ++++
.../examples/SharedRDDExampleSelfTest.java | 36 ------
.../IgniteExamplesJ8SelfTestSuite.java | 2 -
.../examples/SharedRDDExampleSelfTest.java | 36 ++++++
.../IgniteExamplesSparkSelfTestSuite.java | 46 +++++++
9 files changed, 246 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
[04/50] [abbrv] ignite git commit: IGNITE-4169: SQL: implemented
streaming support for INSERT operations. This closes #1350. This closes
#1553.
Posted by av...@apache.org.
IGNITE-4169: SQL: implemented streaming support for INSERT operations. This closes #1350. This closes #1553.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0130b097
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0130b097
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0130b097
Branch: refs/heads/master
Commit: 0130b097b0a24950f369cc5e009a9394c3fd4e1f
Parents: 1a08ef7
Author: Alexander Paschenko <al...@gmail.com>
Authored: Mon Feb 20 13:32:19 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Mon Feb 20 13:32:19 2017 +0300
----------------------------------------------------------------------
.../jdbc2/JdbcAbstractDmlStatementSelfTest.java | 49 +----
.../jdbc2/JdbcInsertStatementSelfTest.java | 51 +++++
.../jdbc2/JdbcMergeStatementSelfTest.java | 51 +++++
.../internal/jdbc2/JdbcStreamingSelfTest.java | 189 ++++++++++++++++
.../jdbc2/JdbcUpdateStatementSelfTest.java | 50 +++++
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 1 +
.../org/apache/ignite/IgniteJdbcDriver.java | 30 +++
.../ignite/internal/jdbc2/JdbcConnection.java | 72 +++++-
.../internal/jdbc2/JdbcPreparedStatement.java | 34 ++-
.../ignite/internal/jdbc2/JdbcStatement.java | 20 +-
.../jdbc2/JdbcStreamedPreparedStatement.java | 59 +++++
.../processors/query/GridQueryIndexing.java | 35 +++
.../processors/query/GridQueryProcessor.java | 63 +++++-
.../query/h2/DmlStatementsProcessor.java | 219 +++++++++++++------
.../processors/query/h2/IgniteH2Indexing.java | 57 ++++-
15 files changed, 812 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
index 4a97aef..332bbba 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
@@ -20,8 +20,6 @@ package org.apache.ignite.internal.jdbc2;
import java.io.Serializable;
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.Statement;
import java.util.Collections;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.QueryEntity;
@@ -54,7 +52,7 @@ public abstract class JdbcAbstractDmlStatementSelfTest extends GridCommonAbstrac
static final String BASE_URL_BIN = CFG_URL_PREFIX + "modules/clients/src/test/config/jdbc-bin-config.xml";
/** SQL SELECT query for verification. */
- private static final String SQL_SELECT = "select _key, id, firstName, lastName, age from Person";
+ static final String SQL_SELECT = "select _key, id, firstName, lastName, age from Person";
/** Connection. */
protected Connection conn;
@@ -149,51 +147,6 @@ public abstract class JdbcAbstractDmlStatementSelfTest extends GridCommonAbstrac
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
- try (Statement selStmt = conn.createStatement()) {
- assert selStmt.execute(SQL_SELECT);
-
- ResultSet rs = selStmt.getResultSet();
-
- assert rs != null;
-
- while (rs.next()) {
- int id = rs.getInt("id");
-
- switch (id) {
- case 1:
- assertEquals("p1", rs.getString("_key"));
- assertEquals("John", rs.getString("firstName"));
- assertEquals("White", rs.getString("lastName"));
- assertEquals(25, rs.getInt("age"));
- break;
-
- case 2:
- assertEquals("p2", rs.getString("_key"));
- assertEquals("Joe", rs.getString("firstName"));
- assertEquals("Black", rs.getString("lastName"));
- assertEquals(35, rs.getInt("age"));
- break;
-
- case 3:
- assertEquals("p3", rs.getString("_key"));
- assertEquals("Mike", rs.getString("firstName"));
- assertEquals("Green", rs.getString("lastName"));
- assertEquals(40, rs.getInt("age"));
- break;
-
- case 4:
- assertEquals("p4", rs.getString("_key"));
- assertEquals("Leah", rs.getString("firstName"));
- assertEquals("Grey", rs.getString("lastName"));
- assertEquals(22, rs.getInt("age"));
- break;
-
- default:
- assert false : "Invalid ID: " + id;
- }
- }
- }
-
grid(0).cache(null).clear();
assertEquals(0, grid(0).cache(null).size(CachePeekMode.ALL));
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
index 7fc92de..1bd6d34 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcInsertStatementSelfTest.java
@@ -18,12 +18,14 @@
package org.apache.ignite.internal.jdbc2;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.HashSet;
import java.util.concurrent.Callable;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.testframework.GridTestUtils;
/**
@@ -61,6 +63,55 @@ public class JdbcInsertStatementSelfTest extends JdbcAbstractDmlStatementSelfTes
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
+ try (Statement selStmt = conn.createStatement()) {
+ assertTrue(selStmt.execute(SQL_SELECT));
+
+ ResultSet rs = selStmt.getResultSet();
+
+ assert rs != null;
+
+ while (rs.next()) {
+ int id = rs.getInt("id");
+
+ switch (id) {
+ case 1:
+ assertEquals("p1", rs.getString("_key"));
+ assertEquals("John", rs.getString("firstName"));
+ assertEquals("White", rs.getString("lastName"));
+ assertEquals(25, rs.getInt("age"));
+ break;
+
+ case 2:
+ assertEquals("p2", rs.getString("_key"));
+ assertEquals("Joe", rs.getString("firstName"));
+ assertEquals("Black", rs.getString("lastName"));
+ assertEquals(35, rs.getInt("age"));
+ break;
+
+ case 3:
+ assertEquals("p3", rs.getString("_key"));
+ assertEquals("Mike", rs.getString("firstName"));
+ assertEquals("Green", rs.getString("lastName"));
+ assertEquals(40, rs.getInt("age"));
+ break;
+
+ case 4:
+ assertEquals("p4", rs.getString("_key"));
+ assertEquals("Leah", rs.getString("firstName"));
+ assertEquals("Grey", rs.getString("lastName"));
+ assertEquals(22, rs.getInt("age"));
+ break;
+
+ default:
+ assert false : "Invalid ID: " + id;
+ }
+ }
+ }
+
+ grid(0).cache(null).clear();
+
+ assertEquals(0, grid(0).cache(null).size(CachePeekMode.ALL));
+
super.afterTest();
if (stmt != null && !stmt.isClosed())
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
index ecf6032..3c56c92 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcMergeStatementSelfTest.java
@@ -18,8 +18,10 @@
package org.apache.ignite.internal.jdbc2;
import java.sql.PreparedStatement;
+import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import org.apache.ignite.cache.CachePeekMode;
/**
* MERGE statement test.
@@ -56,6 +58,55 @@ public class JdbcMergeStatementSelfTest extends JdbcAbstractDmlStatementSelfTest
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
+ try (Statement selStmt = conn.createStatement()) {
+ assertTrue(selStmt.execute(SQL_SELECT));
+
+ ResultSet rs = selStmt.getResultSet();
+
+ assert rs != null;
+
+ while (rs.next()) {
+ int id = rs.getInt("id");
+
+ switch (id) {
+ case 1:
+ assertEquals("p1", rs.getString("_key"));
+ assertEquals("John", rs.getString("firstName"));
+ assertEquals("White", rs.getString("lastName"));
+ assertEquals(25, rs.getInt("age"));
+ break;
+
+ case 2:
+ assertEquals("p2", rs.getString("_key"));
+ assertEquals("Joe", rs.getString("firstName"));
+ assertEquals("Black", rs.getString("lastName"));
+ assertEquals(35, rs.getInt("age"));
+ break;
+
+ case 3:
+ assertEquals("p3", rs.getString("_key"));
+ assertEquals("Mike", rs.getString("firstName"));
+ assertEquals("Green", rs.getString("lastName"));
+ assertEquals(40, rs.getInt("age"));
+ break;
+
+ case 4:
+ assertEquals("p4", rs.getString("_key"));
+ assertEquals("Leah", rs.getString("firstName"));
+ assertEquals("Grey", rs.getString("lastName"));
+ assertEquals(22, rs.getInt("age"));
+ break;
+
+ default:
+ assert false : "Invalid ID: " + id;
+ }
+ }
+ }
+
+ grid(0).cache(null).clear();
+
+ assertEquals(0, grid(0).cache(null).size(CachePeekMode.ALL));
+
super.afterTest();
if (stmt != null && !stmt.isClosed())
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
new file mode 100644
index 0000000..5e206ee
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.util.Properties;
+import org.apache.ignite.IgniteJdbcDriver;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteJdbcDriver.CFG_URL_PREFIX;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Data streaming test.
+ */
+public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** JDBC URL. */
+ private static final String BASE_URL = CFG_URL_PREFIX + "modules/clients/src/test/config/jdbc-config.xml";
+
+ /** Connection. */
+ protected Connection conn;
+
+ /** */
+ protected transient IgniteLogger log;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ return getConfiguration0(gridName);
+ }
+
+ /**
+ * @param gridName Grid name.
+ * @return Grid configuration used for starting the grid.
+ * @throws Exception If failed.
+ */
+ private IgniteConfiguration getConfiguration0(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<?,?> cache = defaultCacheConfiguration();
+
+ cache.setCacheMode(PARTITIONED);
+ cache.setBackups(1);
+ cache.setWriteSynchronizationMode(FULL_SYNC);
+ cache.setIndexedTypes(
+ Integer.class, Integer.class
+ );
+
+ cfg.setCacheConfiguration(cache);
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+ disco.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(disco);
+
+ cfg.setConnectorConfiguration(new ConnectorConfiguration());
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ startGridsMultiThreaded(3);
+
+ Class.forName("org.apache.ignite.IgniteJdbcDriver");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * @param allowOverwrite Allow overwriting of existing keys.
+ * @return Connection to use for the test.
+ * @throws Exception if failed.
+ */
+ private Connection createConnection(boolean allowOverwrite) throws Exception {
+ Properties props = new Properties();
+
+ props.setProperty(IgniteJdbcDriver.PROP_STREAMING, "true");
+ props.setProperty(IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ, "500");
+
+ if (allowOverwrite)
+ props.setProperty(IgniteJdbcDriver.PROP_STREAMING_ALLOW_OVERWRITE, "true");
+
+ return DriverManager.getConnection(BASE_URL, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ U.closeQuiet(conn);
+
+ ignite(0).cache(null).clear();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testStreamedInsert() throws Exception {
+ conn = createConnection(false);
+
+ ignite(0).cache(null).put(5, 500);
+
+ PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
+
+ for (int i = 1; i <= 100000; i++) {
+ stmt.setInt(1, i);
+ stmt.setInt(2, i);
+
+ stmt.executeUpdate();
+ }
+
+ // Data is not there yet.
+ assertNull(grid(0).cache(null).get(100000));
+
+ // Let the stream flush.
+ U.sleep(1500);
+
+ // Now let's check it's all there.
+ assertEquals(1, grid(0).cache(null).get(1));
+ assertEquals(100000, grid(0).cache(null).get(100000));
+
+ // 5 should still point to 500.
+ assertEquals(500, grid(0).cache(null).get(5));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testStreamedInsertWithOverwritesAllowed() throws Exception {
+ conn = createConnection(true);
+
+ ignite(0).cache(null).put(5, 500);
+
+ PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
+
+ for (int i = 1; i <= 100000; i++) {
+ stmt.setInt(1, i);
+ stmt.setInt(2, i);
+
+ stmt.executeUpdate();
+ }
+
+ // Data is not there yet.
+ assertNull(grid(0).cache(null).get(100000));
+
+ // Let the stream flush.
+ U.sleep(1500);
+
+ // Now let's check it's all there.
+ assertEquals(1, grid(0).cache(null).get(1));
+ assertEquals(100000, grid(0).cache(null).get(100000));
+
+ // 5 should now point to 5 as we've turned overwriting on.
+ assertEquals(5, grid(0).cache(null).get(5));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcUpdateStatementSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcUpdateStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcUpdateStatementSelfTest.java
new file mode 100644
index 0000000..8ae0e90
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcUpdateStatementSelfTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.sql.SQLException;
+import java.util.Arrays;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.util.typedef.F;
+
+/**
+ *
+ */
+public class JdbcUpdateStatementSelfTest extends JdbcAbstractUpdateStatementSelfTest {
+ /**
+ *
+ */
+ public void testExecute() throws SQLException {
+ conn.createStatement().execute("update Person set firstName = 'Jack' where " +
+ "cast(substring(_key, 2, 1) as int) % 2 = 0");
+
+ assertEquals(Arrays.asList(F.asList("John"), F.asList("Jack"), F.asList("Mike")),
+ jcache(0).query(new SqlFieldsQuery("select firstName from Person order by _key")).getAll());
+ }
+
+ /**
+ *
+ */
+ public void testExecuteUpdate() throws SQLException {
+ conn.createStatement().executeUpdate("update Person set firstName = 'Jack' where " +
+ "cast(substring(_key, 2, 1) as int) % 2 = 0");
+
+ assertEquals(Arrays.asList(F.asList("John"), F.asList("Jack"), F.asList("Mike")),
+ jcache(0).query(new SqlFieldsQuery("select firstName from Person order by _key")).getAll());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index c41b754..7395fcb 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -66,6 +66,7 @@ public class IgniteJdbcDriverTestSuite extends TestSuite {
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcInsertStatementSelfTest.class));
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcBinaryMarshallerInsertStatementSelfTest.class));
suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcDeleteStatementSelfTest.class));
+ suite.addTest(new TestSuite(org.apache.ignite.internal.jdbc2.JdbcStreamingSelfTest.class));
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
index d432c1e4..9790b8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteJdbcDriver.java
@@ -292,6 +292,21 @@ public class IgniteJdbcDriver implements Driver {
/** Distributed joins parameter name. */
private static final String PARAM_DISTRIBUTED_JOINS = "distributedJoins";
+ /** DML streaming parameter name. */
+ private static final String PARAM_STREAMING = "streaming";
+
+ /** DML streaming auto flush frequency. */
+ private static final String PARAM_STREAMING_FLUSH_FREQ = "streamingFlushFrequency";
+
+ /** DML streaming node buffer size. */
+ private static final String PARAM_STREAMING_PER_NODE_BUF_SIZE = "streamingPerNodeBufferSize";
+
+ /** DML streaming parallel operations per node. */
+ private static final String PARAM_STREAMING_PER_NODE_PAR_OPS = "streamingPerNodeParallelOperations";
+
+ /** Whether DML streaming will overwrite existing cache entries. */
+ private static final String PARAM_STREAMING_ALLOW_OVERWRITE = "streamingAllowOverwrite";
+
/** Hostname property name. */
public static final String PROP_HOST = PROP_PREFIX + "host";
@@ -313,6 +328,21 @@ public class IgniteJdbcDriver implements Driver {
/** Distributed joins property name. */
public static final String PROP_DISTRIBUTED_JOINS = PROP_PREFIX + PARAM_DISTRIBUTED_JOINS;
+ /** DML streaming property name. */
+ public static final String PROP_STREAMING = PROP_PREFIX + PARAM_STREAMING;
+
+ /** DML stream auto flush frequency property name. */
+ public static final String PROP_STREAMING_FLUSH_FREQ = PROP_PREFIX + PARAM_STREAMING_FLUSH_FREQ;
+
+ /** DML stream node buffer size property name. */
+ public static final String PROP_STREAMING_PER_NODE_BUF_SIZE = PROP_PREFIX + PARAM_STREAMING_PER_NODE_BUF_SIZE;
+
+ /** DML stream parallel operations per node property name. */
+ public static final String PROP_STREAMING_PER_NODE_PAR_OPS = PROP_PREFIX + PARAM_STREAMING_PER_NODE_PAR_OPS;
+
+ /** Whether DML streaming will overwrite existing cache entries. */
+ public static final String PROP_STREAMING_ALLOW_OVERWRITE = PROP_PREFIX + PARAM_STREAMING_ALLOW_OVERWRITE;
+
/** Cache name property name. */
public static final String PROP_CFG = PROP_PREFIX + "cfg";
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
index 5c4a147..4244602 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcConnection.java
@@ -48,17 +48,21 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteClientDisconnectedException;
import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteJdbcDriver;
import org.apache.ignite.Ignition;
import org.apache.ignite.cluster.ClusterGroup;
import org.apache.ignite.compute.ComputeTaskTimeoutException;
+import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.resources.IgniteInstanceResource;
@@ -73,6 +77,11 @@ import static org.apache.ignite.IgniteJdbcDriver.PROP_COLLOCATED;
import static org.apache.ignite.IgniteJdbcDriver.PROP_DISTRIBUTED_JOINS;
import static org.apache.ignite.IgniteJdbcDriver.PROP_LOCAL;
import static org.apache.ignite.IgniteJdbcDriver.PROP_NODE_ID;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_ALLOW_OVERWRITE;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_FLUSH_FREQ;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_PER_NODE_BUF_SIZE;
+import static org.apache.ignite.IgniteJdbcDriver.PROP_STREAMING_PER_NODE_PAR_OPS;
/**
* JDBC connection implementation.
@@ -118,6 +127,21 @@ public class JdbcConnection implements Connection {
/** Distributed joins flag. */
private boolean distributedJoins;
+ /** Make this connection streaming oriented, and prepared statements - data streamer aware. */
+ private final boolean stream;
+
+ /** Auto flush frequency for streaming. */
+ private final long streamFlushTimeout;
+
+ /** Node buffer size for data streamer. */
+ private final int streamNodeBufSize;
+
+ /** Parallel ops count per node for data streamer. */
+ private final int streamNodeParOps;
+
+ /** Allow overwrites for duplicate keys on streamed {@code INSERT}s. */
+ private final boolean streamAllowOverwrite;
+
/** Statements. */
final Set<JdbcStatement> statements = new HashSet<>();
@@ -139,6 +163,14 @@ public class JdbcConnection implements Connection {
this.collocatedQry = Boolean.parseBoolean(props.getProperty(PROP_COLLOCATED));
this.distributedJoins = Boolean.parseBoolean(props.getProperty(PROP_DISTRIBUTED_JOINS));
+ stream = Boolean.parseBoolean(props.getProperty(PROP_STREAMING));
+ streamAllowOverwrite = Boolean.parseBoolean(props.getProperty(PROP_STREAMING_ALLOW_OVERWRITE));
+ streamFlushTimeout = Long.parseLong(props.getProperty(PROP_STREAMING_FLUSH_FREQ, "0"));
+ streamNodeBufSize = Integer.parseInt(props.getProperty(PROP_STREAMING_PER_NODE_BUF_SIZE,
+ String.valueOf(IgniteDataStreamer.DFLT_PER_NODE_BUFFER_SIZE)));
+ streamNodeParOps = Integer.parseInt(props.getProperty(PROP_STREAMING_PER_NODE_PAR_OPS,
+ String.valueOf(IgniteDataStreamer.DFLT_MAX_PARALLEL_OPS)));
+
String nodeIdProp = props.getProperty(PROP_NODE_ID);
if (nodeIdProp != null)
@@ -291,6 +323,14 @@ public class JdbcConnection implements Connection {
closed = true;
+ for (Iterator<JdbcStatement> it = statements.iterator(); it.hasNext();) {
+ JdbcStatement stmt = it.next();
+
+ stmt.closeInternal();
+
+ it.remove();
+ }
+
IgniteNodeFuture fut = NODES.get(cfg);
if (fut != null && fut.release()) {
@@ -299,14 +339,6 @@ public class JdbcConnection implements Connection {
if (ignite != null)
ignite.close();
}
-
- for (Iterator<JdbcStatement> it = statements.iterator(); it.hasNext();) {
- JdbcStatement stmt = it.next();
-
- stmt.closeInternal();
-
- it.remove();
- }
}
/** {@inheritDoc} */
@@ -487,7 +519,18 @@ public class JdbcConnection implements Connection {
if (resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
throw new SQLFeatureNotSupportedException("Invalid holdability (transactions are not supported).");
- JdbcPreparedStatement stmt = new JdbcPreparedStatement(this, sql);
+ JdbcPreparedStatement stmt;
+
+ if (!stream)
+ stmt = new JdbcPreparedStatement(this, sql);
+ else {
+ PreparedStatement nativeStmt = prepareNativeStatement(sql);
+
+ IgniteDataStreamer<?, ?> streamer = ((IgniteEx) ignite).context().query().createStreamer(cacheName,
+ nativeStmt, streamFlushTimeout, streamNodeBufSize, streamNodeParOps, streamAllowOverwrite);
+
+ stmt = new JdbcStreamedPreparedStatement(this, sql, streamer, nativeStmt);
+ }
statements.add(stmt);
@@ -646,12 +689,17 @@ public class JdbcConnection implements Connection {
/** {@inheritDoc} */
@Override public void setSchema(String schema) throws SQLException {
- cacheName = schema;
+ assert ignite instanceof IgniteEx;
+
+ cacheName = ((IgniteEx)ignite).context().query().space(schema);
}
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override public String getSchema() throws SQLException {
- return cacheName;
+ String sqlSchema = ignite.cache(cacheName).getConfiguration(CacheConfiguration.class).getSqlSchema();
+
+ return U.firstNotNull(sqlSchema, cacheName, "");
}
/** {@inheritDoc} */
@@ -749,7 +797,7 @@ public class JdbcConnection implements Connection {
*/
PreparedStatement prepareNativeStatement(String sql) throws SQLException {
return ((IgniteCacheProxy) ignite().cache(cacheName())).context()
- .kernalContext().query().prepareNativeStatement(cacheName(), sql);
+ .kernalContext().query().prepareNativeStatement(getSchema(), sql);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
index 57badd2..54e58e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcPreparedStatement.java
@@ -17,12 +17,28 @@
package org.apache.ignite.internal.jdbc2;
-import java.io.*;
-import java.math.*;
-import java.net.*;
-import java.sql.*;
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.net.URL;
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.Clob;
import java.sql.Date;
-import java.util.*;
+import java.sql.NClob;
+import java.sql.ParameterMetaData;
+import java.sql.PreparedStatement;
+import java.sql.Ref;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.RowId;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.sql.SQLXML;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Calendar;
/**
* JDBC prepared statement implementation.
@@ -31,10 +47,8 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
/** SQL query. */
private final String sql;
- /**
- * H2's parsed statement to retrieve metadata from.
- */
- private PreparedStatement nativeStatement;
+ /** H2's parsed statement to retrieve metadata from. */
+ PreparedStatement nativeStatement;
/**
* Creates new prepared statement.
@@ -55,8 +69,6 @@ public class JdbcPreparedStatement extends JdbcStatement implements PreparedStat
throw new SQLFeatureNotSupportedException("Adding new SQL command to batch not supported for prepared statement.");
}
-
-
/** {@inheritDoc} */
@Override public ResultSet executeQuery() throws SQLException {
ensureNotClosed();
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java
index d7e387f..44db375 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStatement.java
@@ -140,7 +140,7 @@ public class JdbcStatement implements Statement {
updateCnt = -1;
- return doUpdate(sql, getArgs());
+ return Long.valueOf(doUpdate(sql, getArgs())).intValue();
}
/**
@@ -148,9 +148,9 @@ public class JdbcStatement implements Statement {
* @param sql SQL query.
* @param args Update arguments.
* @return Number of affected items.
- * @throws SQLException
+ * @throws SQLException If failed.
*/
- int doUpdate(String sql, Object[] args) throws SQLException {
+ long doUpdate(String sql, Object[] args) throws SQLException {
if (F.isEmpty(sql))
throw new SQLException("SQL query is empty");
@@ -172,11 +172,7 @@ public class JdbcStatement implements Statement {
JdbcQueryTaskV2.QueryResult qryRes =
loc ? qryTask.call() : ignite.compute(ignite.cluster().forNodeId(nodeId)).call(qryTask);
- Long res = updateCounterFromQueryResult(qryRes.getRows());
-
- updateCnt = res;
-
- return res.intValue();
+ return updateCnt = updateCounterFromQueryResult(qryRes.getRows());
}
catch (IgniteSQLException e) {
throw e.toJdbcException();
@@ -194,12 +190,12 @@ public class JdbcStatement implements Statement {
* @return update counter, if found
* @throws SQLException if getting an update counter from result proved to be impossible.
*/
- private static Long updateCounterFromQueryResult(List<List<?>> rows) throws SQLException {
+ private static long updateCounterFromQueryResult(List<List<?>> rows) throws SQLException {
if (F.isEmpty(rows))
- return 0L;
+ return -1;
if (rows.size() != 1)
- throw new SQLException("Expected number of rows of 1 for update operation");
+ throw new SQLException("Expected fetch size of 1 for update operation");
List<?> row = rows.get(0);
@@ -211,7 +207,7 @@ public class JdbcStatement implements Statement {
if (!(objRes instanceof Long))
throw new SQLException("Unexpected update result type");
- return (Long) objRes;
+ return (Long)objRes;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
new file mode 100644
index 0000000..019923f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc2/JdbcStreamedPreparedStatement.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.jdbc2;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.IgniteEx;
+
+/**
+ * Prepared statement associated with a data streamer.
+ */
+class JdbcStreamedPreparedStatement extends JdbcPreparedStatement {
+ /** */
+ private final IgniteDataStreamer<?, ?> streamer;
+
+ /**
+ * Creates new prepared statement.
+ *
+ * @param conn Connection.
+ * @param sql SQL query.
+ * @param streamer Data streamer to use with this statement. Will be closed on statement close.
+ */
+ JdbcStreamedPreparedStatement(JdbcConnection conn, String sql, IgniteDataStreamer<?, ?> streamer,
+ PreparedStatement nativeStmt) {
+ super(conn, sql);
+
+ this.streamer = streamer;
+
+ nativeStatement = nativeStmt;
+ }
+
+ /** {@inheritDoc} */
+ @Override void closeInternal() throws SQLException {
+ streamer.close(false);
+
+ super.closeInternal();
+ }
+
+ /** {@inheritDoc} */
+ @Override long doUpdate(String sql, Object[] args) throws SQLException {
+ return ((IgniteEx)conn.ignite()).context().query().streamUpdateQuery(conn.cacheName(), streamer, sql, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index ca04724..2abb3a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.List;
import javax.cache.Cache;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
@@ -99,6 +100,19 @@ public interface GridQueryIndexing {
GridQueryCancel cancel) throws IgniteCheckedException;
/**
+ * Perform a MERGE statement using data streamer as receiver.
+ *
+ * @param spaceName Space name.
+ * @param qry Query.
+ * @param params Query parameters.
+ * @param streamer Data streamer to feed data to.
+ * @return Query result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long streamUpdateQuery(@Nullable final String spaceName, final String qry,
+ @Nullable final Object[] params, IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException;
+
+ /**
* Executes regular query.
*
* @param spaceName Space name.
@@ -241,6 +255,14 @@ public interface GridQueryIndexing {
public PreparedStatement prepareNativeStatement(String schema, String sql) throws SQLException;
/**
+ * Gets space name from database schema.
+ *
+ * @param schemaName Schema name. Could not be null. Could be empty.
+ * @return Space name. Could be null.
+ */
+ public String space(String schemaName);
+
+ /**
* Collect queries that already running more than specified duration.
*
* @param duration Duration to check.
@@ -259,4 +281,17 @@ public interface GridQueryIndexing {
* Cancels all executing queries.
*/
public void cancelAllQueries();
+
+ /**
+ * @param spaceName Space name.
+ * @param nativeStmt Native statement.
+ * @param autoFlushFreq Automatic data flushing frequency, disabled if {@code 0}.
+ * @param nodeBufSize Per node buffer size - see {@link IgniteDataStreamer#perNodeBufferSize(int)}
+ * @param nodeParOps Per node parallel ops count - see {@link IgniteDataStreamer#perNodeParallelOperations(int)}
+ * @param allowOverwrite Overwrite existing cache values on key duplication.
+ * @return {@link IgniteDataStreamer} tailored to specific needs of given native statement based on its metadata;
+ * {@code null} if given statement is a query.
+ */
+ public IgniteDataStreamer<?,?> createStreamer(String spaceName, PreparedStatement nativeStmt, long autoFlushFreq,
+ int nodeBufSize, int nodeParOps, boolean allowOverwrite);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index ee9224b..c6d8270 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -17,12 +17,11 @@
package org.apache.ignite.internal.processors.query;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.concurrent.TimeUnit;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.math.BigDecimal;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.ArrayList;
@@ -40,9 +39,11 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
import javax.cache.Cache;
import javax.cache.CacheException;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.binary.BinaryField;
import org.apache.ignite.binary.BinaryObject;
@@ -818,6 +819,36 @@ public class GridQueryProcessor extends GridProcessorAdapter {
}
/**
+ * @param spaceName Cache name.
+ * @param streamer Data streamer.
+ * @param qry Query.
+ * @return Iterator.
+ */
+ public long streamUpdateQuery(@Nullable final String spaceName,
+ final IgniteDataStreamer<?, ?> streamer, final String qry, final Object[] args) {
+ assert streamer != null;
+
+ if (!busyLock.enterBusy())
+ throw new IllegalStateException("Failed to execute query (grid is stopping).");
+
+ try {
+ GridCacheContext cctx = ctx.cache().cache(spaceName).context();
+
+ return executeQuery(GridCacheQueryType.SQL_FIELDS, qry, cctx, new IgniteOutClosureX<Long>() {
+ @Override public Long applyx() throws IgniteCheckedException {
+ return idx.streamUpdateQuery(spaceName, qry, args, streamer);
+ }
+ }, true);
+ }
+ catch (IgniteCheckedException e) {
+ throw new CacheException(e);
+ }
+ finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
* @param cctx Cache context.
* @param qry Query.
* @return Cursor.
@@ -964,7 +995,6 @@ public class GridQueryProcessor extends GridProcessorAdapter {
}
/**
- *
* @param schema Schema.
* @param sql Query.
* @return {@link PreparedStatement} from underlying engine to supply metadata to Prepared - most likely H2.
@@ -976,6 +1006,31 @@ public class GridQueryProcessor extends GridProcessorAdapter {
}
/**
+ * @param schema Schema name.
+ * @return space (cache) name from schema name.
+ */
+ public String space(String schema) throws SQLException {
+ checkxEnabled();
+
+ return idx.space(schema);
+ }
+
+ /**
+ * @param spaceName Space name.
+ * @param nativeStmt Native statement.
+ * @param autoFlushFreq Automatic data flushing frequency, disabled if {@code 0}.
+ * @param nodeBufSize Per node buffer size - see {@link IgniteDataStreamer#perNodeBufferSize(int)}
+ * @param nodeParOps Per node parallel ops count - see {@link IgniteDataStreamer#perNodeParallelOperations(int)}
+ * @param allowOverwrite Overwrite existing cache values on key duplication.
+ * @see IgniteDataStreamer#allowOverwrite
+ * @return {@link IgniteDataStreamer} tailored to specific needs of given native statement based on its metadata.
+ */
+ public IgniteDataStreamer<?, ?> createStreamer(String spaceName, PreparedStatement nativeStmt, long autoFlushFreq,
+ int nodeBufSize, int nodeParOps, boolean allowOverwrite) {
+ return idx.createStreamer(spaceName, nativeStmt, autoFlushFreq, nodeBufSize, nodeParOps, allowOverwrite);
+ }
+
+ /**
* @param timeout Timeout.
* @param timeUnit Time unit.
* @return Converted time.
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index 4030758..78c5bbc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -21,7 +21,6 @@ import java.lang.reflect.Array;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -40,6 +39,7 @@ import javax.cache.processor.EntryProcessorResult;
import javax.cache.processor.MutableEntry;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.binary.BinaryArrayIdentityResolver;
import org.apache.ignite.binary.BinaryObject;
@@ -60,6 +60,7 @@ import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.h2.dml.FastUpdateArguments;
import org.apache.ignite.internal.processors.query.h2.dml.KeyValueSupplier;
+import org.apache.ignite.internal.processors.query.h2.dml.UpdateMode;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
@@ -126,7 +127,7 @@ public class DmlStatementsProcessor {
* @return Update result (modified items count and failed keys).
* @throws IgniteCheckedException if failed.
*/
- private long updateSqlFields(String spaceName, PreparedStatement stmt, SqlFieldsQuery fieldsQry,
+ private UpdateResult updateSqlFields(String spaceName, PreparedStatement stmt, SqlFieldsQuery fieldsQry,
boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel) throws IgniteCheckedException {
Object[] errKeys = null;
@@ -156,23 +157,27 @@ public class DmlStatementsProcessor {
UpdateResult r;
try {
- r = executeUpdateStatement(cctx, stmt, fieldsQry, loc, filters,
- cancel, errKeys);
+ r = executeUpdateStatement(cctx, stmt, fieldsQry, loc, filters, cancel, errKeys);
}
finally {
cctx.operationContextPerCall(opCtx);
}
- if (F.isEmpty(r.errKeys))
- return r.cnt + items;
- else {
- items += r.cnt;
- errKeys = r.errKeys;
- }
+ items += r.cnt;
+ errKeys = r.errKeys;
+
+ if (F.isEmpty(errKeys))
+ break;
}
- throw new IgniteSQLException("Failed to update or delete some keys: " + Arrays.deepToString(errKeys),
- IgniteQueryErrorCode.CONCURRENT_UPDATE);
+ if (F.isEmpty(errKeys)) {
+ if (items == 1L)
+ return UpdateResult.ONE;
+ else if (items == 0L)
+ return UpdateResult.ZERO;
+ }
+
+ return new UpdateResult(items, errKeys);
}
/**
@@ -186,9 +191,14 @@ public class DmlStatementsProcessor {
@SuppressWarnings("unchecked")
QueryCursorImpl<List<?>> updateSqlFieldsTwoStep(String spaceName, PreparedStatement stmt,
SqlFieldsQuery fieldsQry, GridQueryCancel cancel) throws IgniteCheckedException {
- long res = updateSqlFields(spaceName, stmt, fieldsQry, false, null, cancel);
+ UpdateResult res = updateSqlFields(spaceName, stmt, fieldsQry, false, null, cancel);
+
+ QueryCursorImpl<List<?>> resCur = (QueryCursorImpl<List<?>>)new QueryCursorImpl(Collections.singletonList
+ (Collections.singletonList(res.cnt)), null, false);
- return cursorForUpdateResult(res);
+ resCur.fieldsMeta(UPDATE_RESULT_META);
+
+ return resCur;
}
/**
@@ -203,10 +213,95 @@ public class DmlStatementsProcessor {
@SuppressWarnings("unchecked")
GridQueryFieldsResult updateLocalSqlFields(String spaceName, PreparedStatement stmt,
SqlFieldsQuery fieldsQry, IndexingQueryFilter filters, GridQueryCancel cancel) throws IgniteCheckedException {
- long res = updateSqlFields(spaceName, stmt, fieldsQry, true, filters, cancel);
+ UpdateResult res = updateSqlFields(spaceName, stmt, fieldsQry, true, filters, cancel);
return new GridQueryFieldsResultAdapter(UPDATE_RESULT_META,
- new IgniteSingletonIterator(Collections.singletonList(res)));
+ new IgniteSingletonIterator(Collections.singletonList(res.cnt)));
+ }
+
+ /**
+ * Perform given statement against given data streamer. Only rows based INSERT and MERGE are supported
+ * as well as key bound UPDATE and DELETE (ones with filter {@code WHERE _key = ?}).
+ *
+ * @param streamer Streamer to feed data to.
+ * @param stmt Statement.
+ * @param args Statement arguments.
+ * @return Number of rows in given statement for INSERT and MERGE, {@code 1} otherwise.
+ * @throws IgniteCheckedException if failed.
+ */
+ @SuppressWarnings({"unchecked", "ConstantConditions"})
+ long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, Object[] args)
+ throws IgniteCheckedException {
+ args = U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY);
+
+ Prepared p = GridSqlQueryParser.prepared((JdbcPreparedStatement)stmt);
+
+ assert p != null;
+
+ UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, null);
+
+ if (!F.eq(streamer.cacheName(), plan.tbl.rowDescriptor().context().namex()))
+ throw new IgniteSQLException("Cross cache streaming is not supported, please specify cache explicitly" +
+ " in connection options", IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+ if (plan.mode == UpdateMode.INSERT && plan.rowsNum > 0) {
+ assert plan.isLocSubqry;
+
+ final GridCacheContext cctx = plan.tbl.rowDescriptor().context();
+
+ QueryCursorImpl<List<?>> cur;
+
+ final ArrayList<List<?>> data = new ArrayList<>(plan.rowsNum);
+
+ final GridQueryFieldsResult res = indexing.queryLocalSqlFields(cctx.name(), plan.selectQry,
+ F.asList(args), null, false, 0, null);
+
+ QueryCursorImpl<List<?>> stepCur = new QueryCursorImpl<>(new Iterable<List<?>>() {
+ @Override public Iterator<List<?>> iterator() {
+ try {
+ return new GridQueryCacheObjectsIterator(res.iterator(), cctx, cctx.keepBinary());
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+ }, null);
+
+ data.addAll(stepCur.getAll());
+
+ cur = new QueryCursorImpl<>(new Iterable<List<?>>() {
+ @Override public Iterator<List<?>> iterator() {
+ return data.iterator();
+ }
+ }, null);
+
+ GridH2RowDescriptor desc = plan.tbl.rowDescriptor();
+
+ if (plan.rowsNum == 1) {
+ IgniteBiTuple t = rowToKeyValue(cctx, cur.iterator().next().toArray(), plan.colNames, plan.colTypes,
+ plan.keySupplier, plan.valSupplier, plan.keyColIdx, plan.valColIdx, desc);
+
+ streamer.addData(t.getKey(), t.getValue());
+
+ return 1;
+ }
+
+ Map<Object, Object> rows = new LinkedHashMap<>(plan.rowsNum);
+
+ for (List<?> row : cur) {
+ final IgniteBiTuple t = rowToKeyValue(cctx, row.toArray(), plan.colNames, plan.colTypes,
+ plan.keySupplier, plan.valSupplier, plan.keyColIdx, plan.valColIdx, desc);
+
+ rows.put(t.getKey(), t.getValue());
+ }
+
+ streamer.addData(rows);
+
+ return rows.size();
+ }
+ else
+ throw new IgniteSQLException("Only tuple based INSERT statements are supported in streaming mode",
+ IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
}
/**
@@ -214,31 +309,21 @@ public class DmlStatementsProcessor {
* @param cctx Cache context.
* @param prepStmt Prepared statement for DML query.
* @param filters Space name and key filter.
- * @param failedKeys Keys to restrict UPDATE and DELETE operations with. Null or empty array means no restriction.
- * @return Pair [number of successfully processed items; keys that have failed to be processed]
+ * @param failedKeys Keys to restrict UPDATE and DELETE operations with. Null or empty array means no restriction. @return Pair [number of successfully processed items; keys that have failed to be processed]
* @throws IgniteCheckedException if failed.
*/
- @SuppressWarnings("ConstantConditions")
+ @SuppressWarnings({"ConstantConditions", "unchecked"})
private UpdateResult executeUpdateStatement(final GridCacheContext cctx, PreparedStatement prepStmt,
- SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel, Object[] failedKeys)
- throws IgniteCheckedException {
+ SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel,
+ Object[] failedKeys) throws IgniteCheckedException {
Integer errKeysPos = null;
- Object[] params = fieldsQry.getArgs();
-
- if (!F.isEmpty(failedKeys)) {
- int paramsCnt = F.isEmpty(params) ? 0 : params.length;
- params = Arrays.copyOf(U.firstNotNull(params, X.EMPTY_OBJECT_ARRAY), paramsCnt + 1);
- params[paramsCnt] = failedKeys;
- errKeysPos = paramsCnt; // Last position
- }
-
UpdatePlan plan = getPlanForStatement(cctx.name(), prepStmt, errKeysPos);
if (plan.fastUpdateArgs != null) {
assert F.isEmpty(failedKeys) && errKeysPos == null;
- return new UpdateResult(doSingleUpdate(plan, params), X.EMPTY_OBJECT_ARRAY);
+ return doFastUpdate(plan, fieldsQry.getArgs());
}
assert !F.isEmpty(plan.selectQry);
@@ -249,7 +334,7 @@ public class DmlStatementsProcessor {
// subquery and not some dummy stuff like "select 1, 2, 3;"
if (!loc && !plan.isLocSubqry) {
SqlFieldsQuery newFieldsQry = new SqlFieldsQuery(plan.selectQry, fieldsQry.isCollocated())
- .setArgs(params)
+ .setArgs(fieldsQry.getArgs())
.setDistributedJoins(fieldsQry.isDistributedJoins())
.setEnforceJoinOrder(fieldsQry.isEnforceJoinOrder())
.setLocal(fieldsQry.isLocal())
@@ -259,8 +344,8 @@ public class DmlStatementsProcessor {
cur = (QueryCursorImpl<List<?>>) indexing.queryTwoStep(cctx, newFieldsQry, cancel);
}
else {
- final GridQueryFieldsResult res = indexing.queryLocalSqlFields(cctx.name(), plan.selectQry, F.asList(params),
- filters, fieldsQry.isEnforceJoinOrder(), fieldsQry.getTimeout(), cancel);
+ final GridQueryFieldsResult res = indexing.queryLocalSqlFields(cctx.name(), plan.selectQry,
+ F.asList(fieldsQry.getArgs()), filters, fieldsQry.isEnforceJoinOrder(), fieldsQry.getTimeout(), cancel);
cur = new QueryCursorImpl<>(new Iterable<List<?>>() {
@Override public Iterator<List<?>> iterator() {
@@ -272,8 +357,6 @@ public class DmlStatementsProcessor {
}
}
}, cancel);
-
- cur.fieldsMeta(res.metaData());
}
int pageSize = loc ? 0 : fieldsQry.getPageSize();
@@ -337,38 +420,41 @@ public class DmlStatementsProcessor {
/**
* Perform single cache operation based on given args.
- * @param params Query parameters.
+ * @param args Query parameters.
* @return 1 if an item was affected, 0 otherwise.
* @throws IgniteCheckedException if failed.
*/
@SuppressWarnings({"unchecked", "ConstantConditions"})
- private static long doSingleUpdate(UpdatePlan plan, Object[] params) throws IgniteCheckedException {
+ private static UpdateResult doFastUpdate(UpdatePlan plan, Object[] args) throws IgniteCheckedException {
GridCacheContext cctx = plan.tbl.rowDescriptor().context();
FastUpdateArguments singleUpdate = plan.fastUpdateArgs;
assert singleUpdate != null;
- int res;
+ boolean valBounded = (singleUpdate.val != FastUpdateArguments.NULL_ARGUMENT);
- Object key = singleUpdate.key.apply(params);
- Object val = singleUpdate.val.apply(params);
- Object newVal = singleUpdate.newVal.apply(params);
+ if (singleUpdate.newVal != FastUpdateArguments.NULL_ARGUMENT) { // Single item UPDATE
+ Object key = singleUpdate.key.apply(args);
+ Object newVal = singleUpdate.newVal.apply(args);
- if (newVal != null) { // Single item UPDATE
- if (val == null) // No _val bound in source query
- res = cctx.cache().replace(key, newVal) ? 1 : 0;
+ if (valBounded) {
+ Object val = singleUpdate.val.apply(args);
+
+ return (cctx.cache().replace(key, val, newVal) ? UpdateResult.ONE : UpdateResult.ZERO);
+ }
else
- res = cctx.cache().replace(key, val, newVal) ? 1 : 0;
+ return (cctx.cache().replace(key, newVal) ? UpdateResult.ONE : UpdateResult.ZERO);
}
else { // Single item DELETE
- if (val == null) // No _val bound in source query
- res = cctx.cache().remove(key) ? 1 : 0;
+ Object key = singleUpdate.key.apply(args);
+ Object val = singleUpdate.val.apply(args);
+
+ if (singleUpdate.val == FastUpdateArguments.NULL_ARGUMENT) // No _val bound in source query
+ return cctx.cache().remove(key) ? UpdateResult.ONE : UpdateResult.ZERO;
else
- res = cctx.cache().remove(key, val) ? 1 : 0;
+ return cctx.cache().remove(key, val) ? UpdateResult.ONE : UpdateResult.ZERO;
}
-
- return res;
}
/**
@@ -379,7 +465,7 @@ public class DmlStatementsProcessor {
* @return Results of DELETE (number of items affected AND keys that failed to be updated).
*/
@SuppressWarnings({"unchecked", "ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
- private UpdateResult doDelete(GridCacheContext cctx, QueryCursorImpl<List<?>> cursor, int pageSize)
+ private UpdateResult doDelete(GridCacheContext cctx, Iterable<List<?>> cursor, int pageSize)
throws IgniteCheckedException {
// With DELETE, we have only two columns - key and value.
long res = 0;
@@ -449,7 +535,7 @@ public class DmlStatementsProcessor {
* had been modified concurrently (arguments for a re-run)].
*/
@SuppressWarnings({"unchecked", "ThrowableResultOfMethodCallIgnored"})
- private UpdateResult doUpdate(UpdatePlan plan, QueryCursorImpl<List<?>> cursor, int pageSize)
+ private UpdateResult doUpdate(UpdatePlan plan, Iterable<List<?>> cursor, int pageSize)
throws IgniteCheckedException {
GridH2RowDescriptor desc = plan.tbl.rowDescriptor();
@@ -575,7 +661,6 @@ public class DmlStatementsProcessor {
throw new IgniteSQLException(resEx);
}
-
return new UpdateResult(res, failedKeys.toArray());
}
@@ -689,7 +774,7 @@ public class DmlStatementsProcessor {
* @throws IgniteCheckedException if failed.
*/
@SuppressWarnings("unchecked")
- private long doMerge(UpdatePlan plan, QueryCursorImpl<List<?>> cursor, int pageSize) throws IgniteCheckedException {
+ private long doMerge(UpdatePlan plan, Iterable<List<?>> cursor, int pageSize) throws IgniteCheckedException {
GridH2RowDescriptor desc = plan.tbl.rowDescriptor();
GridCacheContext cctx = desc.context();
@@ -735,7 +820,7 @@ public class DmlStatementsProcessor {
* @throws IgniteCheckedException if failed, particularly in case of duplicate keys.
*/
@SuppressWarnings({"unchecked", "ConstantConditions"})
- private long doInsert(UpdatePlan plan, QueryCursorImpl<List<?>> cursor, int pageSize) throws IgniteCheckedException {
+ private long doInsert(UpdatePlan plan, Iterable<List<?>> cursor, int pageSize) throws IgniteCheckedException {
GridH2RowDescriptor desc = plan.tbl.rowDescriptor();
GridCacheContext cctx = desc.context();
@@ -999,24 +1084,14 @@ public class DmlStatementsProcessor {
}
}
- /**
- * Wrap result of DML operation (number of items affected) to Iterable suitable to be wrapped by cursor.
- *
- * @param itemsCnt Update result to wrap.
- * @return Resulting Iterable.
- */
- @SuppressWarnings("unchecked")
- private static QueryCursorImpl<List<?>> cursorForUpdateResult(long itemsCnt) {
- QueryCursorImpl<List<?>> res =
- new QueryCursorImpl(Collections.singletonList(Collections.singletonList(itemsCnt)), null, false);
-
- res.fieldsMeta(UPDATE_RESULT_META);
-
- return res;
- }
-
/** Update result - modifications count and keys to re-run query with, if needed. */
private final static class UpdateResult {
+ /** Result to return for operations that affected 1 item - mostly to be used for fast updates and deletes. */
+ final static UpdateResult ONE = new UpdateResult(1, X.EMPTY_OBJECT_ARRAY);
+
+ /** Result to return for operations that affected 0 items - mostly to be used for fast updates and deletes. */
+ final static UpdateResult ZERO = new UpdateResult(0, X.EMPTY_OBJECT_ARRAY);
+
/** Number of processed items. */
final long cnt;
http://git-wip-us.apache.org/repos/asf/ignite/blob/0130b097/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index e4b0c1f..8088f80 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -56,6 +56,7 @@ import java.util.concurrent.atomic.AtomicLong;
import javax.cache.Cache;
import javax.cache.CacheException;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheMemoryMode;
@@ -82,7 +83,6 @@ import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
-import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.GridQueryCancel;
import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
@@ -91,6 +91,7 @@ import org.apache.ignite.internal.processors.query.GridQueryIndexDescriptor;
import org.apache.ignite.internal.processors.query.GridQueryIndexing;
import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2DefaultTableEngine;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOffheap;
@@ -136,6 +137,7 @@ import org.h2.api.ErrorCode;
import org.h2.api.JavaObjectSerializer;
import org.h2.command.CommandInterface;
import org.h2.command.Prepared;
+import org.h2.command.dml.Insert;
import org.h2.engine.Session;
import org.h2.engine.SysProperties;
import org.h2.index.Index;
@@ -440,7 +442,32 @@ public class IgniteH2Indexing implements GridQueryIndexing {
/** {@inheritDoc} */
@Override public PreparedStatement prepareNativeStatement(String schema, String sql) throws SQLException {
- return prepareStatement(connectionForSpace(schema), sql, false);
+ return prepareStatement(connectionForSpace(space(schema)), sql, true);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public IgniteDataStreamer<?, ?> createStreamer(String spaceName, PreparedStatement nativeStmt,
+ long autoFlushFreq, int nodeBufSize, int nodeParOps, boolean allowOverwrite) {
+ Prepared prep = GridSqlQueryParser.prepared((JdbcPreparedStatement) nativeStmt);
+
+ if (!(prep instanceof Insert))
+ throw new IgniteSQLException("Only INSERT operations are supported in streaming mode",
+ IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+ IgniteDataStreamer streamer = ctx.grid().dataStreamer(spaceName);
+
+ streamer.autoFlushFrequency(autoFlushFreq);
+
+ streamer.allowOverwrite(allowOverwrite);
+
+ if (nodeBufSize > 0)
+ streamer.perNodeBufferSize(nodeBufSize);
+
+ if (nodeParOps > 0)
+ streamer.perNodeParallelOperations(nodeParOps);
+
+ return streamer;
}
/**
@@ -873,6 +900,23 @@ public class IgniteH2Indexing implements GridQueryIndexing {
};
}
+ /** {@inheritDoc} */
+ @Override public long streamUpdateQuery(@Nullable String spaceName, String qry,
+ @Nullable Object[] params, IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException {
+ final Connection conn = connectionForSpace(spaceName);
+
+ final PreparedStatement stmt;
+
+ try {
+ stmt = prepareStatement(conn, qry, true);
+ }
+ catch (SQLException e) {
+ throw new IgniteSQLException(e);
+ }
+
+ return dmlProc.streamUpdateQuery(streamer, stmt, params);
+ }
+
/**
* @param rsMeta Metadata.
* @return List of fields metadata.
@@ -1680,13 +1724,8 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
}
- /**
- * Gets space name from database schema.
- *
- * @param schemaName Schema name. Could not be null. Could be empty.
- * @return Space name. Could be null.
- */
- public String space(String schemaName) {
+ /** {@inheritDoc} */
+ @Override public String space(String schemaName) {
assert schemaName != null;
Schema schema = schemas.get(schemaName);
[28/50] [abbrv] ignite git commit: ignite-3860 - fixes2
Posted by av...@apache.org.
ignite-3860 - fixes2
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bf337bc5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bf337bc5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bf337bc5
Branch: refs/heads/master
Commit: bf337bc53bca0a1a0ab6b8b5d4f19e509287ebe9
Parents: 207a669
Author: Sergi Vladykin <se...@gmail.com>
Authored: Tue Feb 21 18:53:16 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Tue Feb 21 18:53:16 2017 +0300
----------------------------------------------------------------------
.../query/h2/sql/GridSqlQuerySplitter.java | 52 ++++++++++++------
.../query/h2/sql/GridSqlStatement.java | 5 ++
.../query/h2/sql/GridSqlSubquery.java | 2 +-
.../h2/sql/AbstractH2CompareQueryTest.java | 21 +++-----
.../query/h2/sql/H2CompareBigQueryTest.java | 56 +++++++++++++-------
.../processors/query/h2/sql/bigQuery.sql | 2 +-
6 files changed, 86 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 165b6b8..a215f20 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -21,6 +21,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -35,7 +36,9 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.h2.command.Prepared;
import org.h2.command.dml.Query;
import org.h2.command.dml.SelectUnion;
@@ -407,15 +410,16 @@ public class GridSqlQuerySplitter {
* @param qrym Query model.
*/
private void injectSortingFirstJoin(QueryModel qrym) {
- // Must always be generated unique aliases here.
- GridSqlAlias leftTbl = qrym.get(0).ast();
- GridSqlAlias rightTbl = qrym.get(1).ast();
-
GridSqlJoin join = findJoin(qrym, 0);
// We are at the beginning, thus left and right AST must be children of the same join AST.
- assert join.leftTable() == leftTbl: join.getSQL();
- assert join.rightTable() == rightTbl: join.getSQL();
+ // join2
+ // / \
+ // join1 \
+ // / \ \
+ // T0 T1 T2
+ GridSqlAlias leftTbl = (GridSqlAlias)join.leftTable();
+ GridSqlAlias rightTbl = (GridSqlAlias)join.rightTable();
// Collect all AND conditions.
List<AndCondition> andConditions = new ArrayList<>();
@@ -461,10 +465,15 @@ public class GridSqlQuerySplitter {
* @param orderByCols Columns for ORDER BY.
*/
private void injectOrderBy(GridSqlAlias subQryAlias, List<GridSqlColumn> orderByCols) {
+ if (orderByCols.isEmpty())
+ return;
+
// Structure: alias -> subquery -> query
- GridSqlQuery qry = subQryAlias.child().child();
+ GridSqlQuery qry = GridSqlAlias.<GridSqlSubquery>unwrap(subQryAlias).subquery();
GridSqlSelect select = leftmostSelect(qry); // The leftmost SELECT in UNION defines column names.
+ BitSet set = new BitSet();
+
for (int i = 0; i < orderByCols.size(); i++) {
GridSqlColumn col = orderByCols.get(i);
@@ -480,12 +489,21 @@ public class GridSqlQuerySplitter {
else if (expr instanceof GridSqlColumn)
colName = ((GridSqlColumn)expr).columnName();
else
- throw new IllegalStateException(); // It must be impossible to join by this column then.
+ // It must be impossible to join by this column then, because the expression has no name.
+ throw new IllegalStateException();
if (colName.equals(col.columnName()))
break; // Found the needed column index.
+
+ colIdx++;
}
+ // Avoid duplicates.
+ if (set.get(colIdx))
+ continue;
+
+ set.set(colIdx, true);
+
// Add sort column to the query.
qry.addSort(new GridSqlSortColumn(colIdx, true, false, false));
}
@@ -591,8 +609,6 @@ public class GridSqlQuerySplitter {
for (int i = begin; i <= end; i++) {
QueryModel child = qrym.get(i);
- assert !child.needSplit && !child.needSplitChild;
-
wrapQrym.add(child);
}
@@ -620,7 +636,7 @@ public class GridSqlQuerySplitter {
int end,
GridSqlAlias wrapAlias
) {
- GridSqlSelect wrapSelect = wrapAlias.child();
+ GridSqlSelect wrapSelect = GridSqlAlias.<GridSqlSubquery>unwrap(wrapAlias).subquery();
final int last = qrym.size() - 1;
@@ -887,7 +903,7 @@ public class GridSqlQuerySplitter {
if (select.where() == null)
return;
- GridSqlSelect wrapSelect = wrapAlias.child();
+ GridSqlSelect wrapSelect = GridSqlAlias.<GridSqlSubquery>unwrap(wrapAlias).subquery();
List<AndCondition> andConditions = new ArrayList<>();
@@ -932,7 +948,7 @@ public class GridSqlQuerySplitter {
}
else {
for (int i = 0; i < ast.size(); i++) {
- if (!isAllRelatedToTables(tblAliases, ast))
+ if (!isAllRelatedToTables(tblAliases, ast.child(i)))
return false;
}
}
@@ -995,8 +1011,6 @@ public class GridSqlQuerySplitter {
for (int i = qrym.size() - 1; i > idx; i--)
join = (GridSqlJoin)join.leftTable();
- assert join.rightTable() == qrym.get(idx).ast();
-
return join;
}
@@ -1956,6 +1970,7 @@ public class GridSqlQuerySplitter {
*/
private static final class QueryModel extends ArrayList<QueryModel> {
/** */
+ @GridToStringInclude
final Type type;
/** */
@@ -1968,9 +1983,11 @@ public class GridSqlQuerySplitter {
int childIdx;
/** If it is a SELECT and we need to split it. Makes sense only for type SELECT. */
+ @GridToStringInclude
boolean needSplit;
/** If we have a child SELECT that we should split. */
+ @GridToStringInclude
boolean needSplitChild;
/** If this is UNION ALL. Makes sense only for type UNION.*/
@@ -2003,6 +2020,11 @@ public class GridSqlQuerySplitter {
private boolean isQuery() {
return type == Type.SELECT || type == Type.UNION;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(QueryModel.class, this);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
index 21cf596..f3acfb9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
@@ -31,6 +31,11 @@ public abstract class GridSqlStatement {
*/
public abstract String getSQL();
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return getSQL();
+ }
+
/**
* @param explain Explain.
* @return {@code this}.
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
index 887e427..f073e7d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
@@ -41,7 +41,7 @@ public class GridSqlSubquery extends GridSqlElement {
/**
* @return Subquery AST.
*/
- public GridSqlQuery subquery() {
+ public <X extends GridSqlQuery> X subquery() {
return child(0);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
index f0ab123..07c8c5e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
@@ -25,12 +25,10 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import junit.framework.AssertionFailedError;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
@@ -39,7 +37,6 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -316,15 +313,7 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest
setDistributedJoins(distrib).
setEnforceJoinOrder(enforceJoinOrder)).getAll();
- try {
- assertRsEquals(h2Res, cacheRes, ordering);
- }
- catch (AssertionFailedError e) {
- X.println("Sql query:\n" + sql + "\nargs=" + Arrays.toString(args));
- X.println("[h2Res=" + h2Res + ", cacheRes=" + cacheRes + "]");
-
- throw e;
- }
+ assertRsEquals(h2Res, cacheRes, ordering);
return h2Res;
}
@@ -407,7 +396,9 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest
Iterator<Map.Entry<String,Integer>> iter1 = rowsWithCnt1.entrySet().iterator();
Iterator<Map.Entry<String,Integer>> iter2 = rowsWithCnt2.entrySet().iterator();
- for (;;) {
+ int uSize = rowsWithCnt1.size();
+
+ for (int i = 0;; i++) {
if (!iter1.hasNext()) {
assertFalse(iter2.hasNext());
@@ -419,8 +410,8 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest
Map.Entry<String, Integer> e1 = iter1.next();
Map.Entry<String, Integer> e2 = iter2.next();
- assertEquals(e1.getKey(), e2.getKey());
- assertEquals(e1.getValue(), e2.getValue());
+ assertEquals("Key " + i + " of " + uSize, e1.getKey(), e2.getKey());
+ assertEquals("Count " + i + " of " + uSize, e1.getValue(), e2.getValue());
}
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
index dc73e96..f8526a8 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
@@ -31,8 +31,10 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.affinity.AffinityKey;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.typedef.X;
/**
* Executes one big query (and subqueries of the big query) to compare query results from h2 database instance and
@@ -54,7 +56,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
* - date - | - date - |
* - alias - | - alias - |
* - archSeq - | - archSeq - ------------------- |
- * ------------------ | --------------------- ----repl.Exec------ |
+ * ------------------ | --------------------- ----part.Exec------ |
* | ------------------- |
* ----------------- | - rootOrderId PK - ----
* ---part.Cancel--- | - date -
@@ -119,19 +121,15 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
/** {@inheritDoc} */
@Override protected void setIndexedTypes(CacheConfiguration<?, ?> cc, CacheMode mode) {
- if (mode == CacheMode.PARTITIONED)
+ if (mode == CacheMode.PARTITIONED) {
cc.setIndexedTypes(
Integer.class, CustOrder.class,
useColocatedData() ? AffinityKey.class : Integer.class, ReplaceOrder.class,
useColocatedData() ? AffinityKey.class : Integer.class, OrderParams.class,
- useColocatedData() ? AffinityKey.class : Integer.class, Cancel.class
+ useColocatedData() ? AffinityKey.class : Integer.class, Cancel.class,
+ useColocatedData() ? AffinityKey.class : Integer.class, Exec.class
);
- else if (mode == CacheMode.REPLICATED)
- cc.setIndexedTypes(
- Integer.class, Exec.class
- );
- else
- throw new IllegalStateException("mode: " + mode);
+ }
}
/** {@inheritDoc} */
@@ -218,11 +216,12 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
int price = 1000 + rootOrderId;
int latsMkt = 3000 + rootOrderId;
- Exec exec = new Exec(rootOrderId, dates.get(rootOrderId % dates.size()), execShares, price, latsMkt);
+ Exec exec = new Exec(idGen.incrementAndGet(), rootOrderId,
+ dates.get(rootOrderId % dates.size()), execShares, price, latsMkt);
add(exec);
- rCache.put(exec.rootOrderId, exec);
+ pCache.put(exec.key(useColocatedData()), exec);
insertInDb(exec);
}
@@ -239,13 +238,20 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
"from \"part\".ReplaceOrder");
compareQueryRes0("select _key, _val, id, date, orderId, parentAlgo from \"part\".OrderParams\n");
compareQueryRes0("select _key, _val, id, date, refOrderId from \"part\".Cancel\n");
- compareQueryRes0(rCache, "select _key, _val, date, rootOrderId, execShares, price, lastMkt from \"repl\".Exec\n");
+ compareQueryRes0("select _key, _val, date, rootOrderId, execShares, price, lastMkt from \"part\".Exec\n");
}
/**
* @throws Exception If failed.
*/
public void testBigQuery() throws Exception {
+ X.println();
+ X.println(bigQry);
+ X.println();
+
+ X.println("Plan: " + pCache.query(new SqlFieldsQuery("EXPLAIN " + bigQry)
+ .setDistributedJoins(distributedJoins())).getAll());
+
List<List<?>> res = compareQueryRes0(pCache, bigQry, distributedJoins(), new Object[0], Ordering.RANDOM);
assertTrue(!res.isEmpty()); // Ensure we set good testing data at database.
@@ -301,9 +307,9 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
" refOrderId int" +
" )");
- st.execute("create table \"repl\".Exec" +
+ st.execute("create table \"part\".Exec" +
" (" +
- " _key int not null," +
+ " _key " + keyType + " not null," +
" _val other not null," +
" rootOrderId int unique," +
" date Date, " +
@@ -414,11 +420,11 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
*/
private void insertInDb(Exec o) throws SQLException {
try(PreparedStatement st = conn.prepareStatement(
- "insert into \"repl\".Exec (_key, _val, date, rootOrderId, execShares, price, lastMkt) " +
+ "insert into \"part\".Exec (_key, _val, date, rootOrderId, execShares, price, lastMkt) " +
"values(?, ?, ?, ?, ?, ?, ?)")) {
int i = 0;
- st.setObject(++i, o.rootOrderId);
+ st.setObject(++i, o.key(useColocatedData()));
st.setObject(++i, o);
st.setObject(++i, o.date);
st.setObject(++i, o.rootOrderId);
@@ -654,8 +660,12 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
* Execute information about root query.
*/
static class Exec implements Serializable {
- /** Primary key. */
+ /** */
@QuerySqlField
+ private int id;
+
+ /** */
+ @QuerySqlField(index = true)
private int rootOrderId;
/** Date */
@@ -675,13 +685,15 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
private int lastMkt;
/**
+ * @param id ID.
* @param rootOrderId Root order id.
* @param date Date.
* @param execShares Execute shares.
* @param price Price.
* @param lastMkt Last mkt.
*/
- Exec(int rootOrderId, Date date, int execShares, int price, int lastMkt) {
+ Exec(int id, int rootOrderId, Date date, int execShares, int price, int lastMkt) {
+ this.id = id;
this.rootOrderId = rootOrderId;
this.date = date;
this.execShares = execShares;
@@ -691,12 +703,16 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
/** {@inheritDoc} */
@Override public boolean equals(Object o) {
- return this == o || o instanceof Exec && rootOrderId == ((Exec)o).rootOrderId;
+ return this == o || o instanceof Exec && id == ((Exec)o).id;
}
/** {@inheritDoc} */
@Override public int hashCode() {
- return rootOrderId;
+ return id;
+ }
+
+ public Object key(boolean useColocatedData) {
+ return useColocatedData ? new AffinityKey<>(id, rootOrderId) : id;
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
index efa768c..8d42d44 100644
--- a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
+++ b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
@@ -46,7 +46,7 @@ inner join (
select e.date, e.rootOrderId as eRootOrderId, e.rootOrderId, sum(e.execShares) as execShares,
sum(e.execShares*e.price)/sum(e.execShares) as price,
case when min(e.lastMkt) = max(e.lastMkt) then min(e.lastMkt) else min('XOFF') end as execMeet
- from "repl".Exec e
+ from "part".Exec e
group by e.date, e.rootOrderId
) oep on (cop.date = oep.date and cop.custOrderId = oep.eRootOrderId)
left outer join (
[16/50] [abbrv] ignite git commit: Added Spatial Index segmentation.
Posted by av...@apache.org.
Added Spatial Index segmentation.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e69d3c34
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e69d3c34
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e69d3c34
Branch: refs/heads/master
Commit: e69d3c34915f2f7de1b66f22e0584b7c09d8fe71
Parents: 64ba13b
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 11:35:34 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 11:52:58 2017 +0300
----------------------------------------------------------------------
.../query/h2/opt/GridH2SpatialIndex.java | 69 +++++++++++++++++---
.../h2/GridH2IndexingSegmentedGeoSelfTest.java | 30 +++++++++
.../testsuites/GeoSpatialIndexingTestSuite.java | 2 +
.../processors/query/h2/IgniteH2Indexing.java | 16 ++---
.../query/h2/opt/GridH2IndexBase.java | 21 +++---
.../query/h2/opt/GridH2TreeIndex.java | 35 ++--------
6 files changed, 118 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
index 3062d13..096b82d 100644
--- a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
+++ b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
@@ -41,7 +41,6 @@ import org.h2.mvstore.rtree.SpatialKey;
import org.h2.result.SearchRow;
import org.h2.result.SortOrder;
import org.h2.table.IndexColumn;
-import org.h2.table.Table;
import org.h2.table.TableFilter;
import org.h2.value.Value;
import org.h2.value.ValueGeometry;
@@ -67,7 +66,7 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
private boolean closed;
/** */
- private final MVRTreeMap<Long> treeMap;
+ private final MVRTreeMap<Long>[] segments;
/** */
private final Map<Long, GridH2Row> idToRow = new HashMap<>();
@@ -83,7 +82,17 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
* @param idxName Index name.
* @param cols Columns.
*/
- public GridH2SpatialIndex(Table tbl, String idxName, IndexColumn... cols) {
+ public GridH2SpatialIndex(GridH2Table tbl, String idxName, IndexColumn... cols) {
+ this(tbl, idxName, 1, cols);
+ }
+
+ /**
+ * @param tbl Table.
+ * @param idxName Index name.
+ * @param segmentsCnt Index segments count.
+ * @param cols Columns.
+ */
+ public GridH2SpatialIndex(GridH2Table tbl, String idxName, int segmentsCnt, IndexColumn... cols) {
if (cols.length > 1)
throw DbException.getUnsupportedException("can only do one column");
@@ -107,7 +116,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
// Index in memory
store = MVStore.open(null);
- treeMap = store.openMap("spatialIndex", new MVRTreeMap.Builder<Long>());
+
+ segments = new MVRTreeMap[segmentsCnt];
+
+ for (int i = 0; i < segmentsCnt; i++)
+ segments[i] = store.openMap("spatialIndex-" + i, new MVRTreeMap.Builder<Long>());
}
/**
@@ -119,6 +132,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
}
/** {@inheritDoc} */
+ @Override protected int segmentsCount() {
+ return segments.length;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override protected Object doTakeSnapshot() {
return null; // TODO We do not support snapshots, but probably this is possible.
}
@@ -140,20 +158,26 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
Long rowId = keyToId.get(key);
+ int seg;
+
if (rowId != null) {
- Long oldRowId = treeMap.remove(getEnvelope(idToRow.get(rowId), rowId));
+ seg = segmentForRowID(rowId);
+
+ Long oldRowId = segments[seg].remove(getEnvelope(idToRow.get(rowId), rowId));
assert rowId.equals(oldRowId);
}
else {
rowId = ++rowIds;
+ seg = segmentForRowID(rowId);
+
keyToId.put(key, rowId);
}
GridH2Row old = idToRow.put(rowId, row);
- treeMap.put(getEnvelope(row, rowId), rowId);
+ segments[seg].put(getEnvelope(row, rowId), rowId);
if (old == null)
rowCnt++; // No replace.
@@ -166,6 +190,17 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
}
/**
+ * @param id Row ID.
+ *
+ * @return Segment ID for given row ID.
+ */
+ private int segmentForRowID(Long id) {
+ assert id != null;
+
+ return (int)(id % segmentsCount());
+ }
+
+ /**
* @param row Row.
* @param rowId Row id.
* @return Envelope.
@@ -200,7 +235,9 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
assert oldRow != null;
- if (!treeMap.remove(getEnvelope(row, rowId), rowId))
+ int seg = segmentForRowID(rowId);
+
+ if (!segments[seg].remove(getEnvelope(row, rowId), rowId))
throw DbException.throwInternalError("row not found");
rowCnt--;
@@ -258,7 +295,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
try {
checkClosed();
- return new GridH2Cursor(rowIterator(treeMap.keySet().iterator(), filter));
+ final int seg = threadLocalSegment();
+
+ final MVRTreeMap<Long> segment = segments[seg];
+
+ return new GridH2Cursor(rowIterator(segment.keySet().iterator(), filter));
}
finally {
l.unlock();
@@ -305,7 +346,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
if (!first)
throw DbException.throwInternalError("Spatial Index can only be fetch by ascending order");
- Iterator<GridH2Row> iter = rowIterator(treeMap.keySet().iterator(), null);
+ final int seg = threadLocalSegment();
+
+ final MVRTreeMap<Long> segment = segments[seg];
+
+ Iterator<GridH2Row> iter = rowIterator(segment.keySet().iterator(), null);
return new SingleRowCursor(iter.hasNext() ? iter.next() : null);
}
@@ -334,7 +379,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
if (intersection == null)
return find(filter.getSession(), null, null);
- return new GridH2Cursor(rowIterator(treeMap.findIntersectingKeys(getEnvelope(intersection, 0)), filter));
+ final int seg = threadLocalSegment();
+
+ final MVRTreeMap<Long> segment = segments[seg];
+
+ return new GridH2Cursor(rowIterator(segment.findIntersectingKeys(getEnvelope(intersection, 0)), filter));
}
finally {
l.unlock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
new file mode 100644
index 0000000..b806321
--- /dev/null
+++ b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+
+/**
+ * Test for segmented geo index.
+ */
+public class GridH2IndexingSegmentedGeoSelfTest extends GridH2IndexingGeoSelfTest {
+ /** {@inheritDoc} */
+ @Override protected CacheConfiguration cacheConfiguration(String gridName) throws Exception {
+ return super.cacheConfiguration(gridName).setQueryParallelism(7);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java b/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
index 1773894..3907b9e 100644
--- a/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
+++ b/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.query.h2.GridBinaryH2IndexingGeoSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexingGeoSelfTest;
+import org.apache.ignite.internal.processors.query.h2.GridH2IndexingSegmentedGeoSelfTest;
/**
* Geospatial indexing tests.
@@ -35,6 +36,7 @@ public class GeoSpatialIndexingTestSuite extends TestSuite {
// Geo.
suite.addTestSuite(GridH2IndexingGeoSelfTest.class);
suite.addTestSuite(GridBinaryH2IndexingGeoSelfTest.class);
+ suite.addTestSuite(GridH2IndexingSegmentedGeoSelfTest.class);
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 2f40d87..652d1f3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2813,7 +2813,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @param cols Columns.
*/
private SpatialIndex createH2SpatialIndex(
- Table tbl,
+ GridH2Table tbl,
String idxName,
IndexColumn[] cols
) {
@@ -2823,14 +2823,17 @@ public class IgniteH2Indexing implements GridQueryIndexing {
Class<?> cls = Class.forName(className);
Constructor<?> ctor = cls.getConstructor(
- Table.class,
+ GridH2Table.class,
String.class,
+ Integer.TYPE,
IndexColumn[].class);
if (!ctor.isAccessible())
ctor.setAccessible(true);
- return (SpatialIndex)ctor.newInstance(tbl, idxName, cols);
+ final int segments = tbl.rowDescriptor().context().config().getQueryParallelism();
+
+ return (SpatialIndex)ctor.newInstance(tbl, idxName, segments, cols);
}
catch (Exception e) {
throw new IgniteException("Failed to instantiate: " + className, e);
@@ -2845,12 +2848,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @return
*/
private Index createTreeIndex(String idxName, GridH2Table tbl, boolean pk, List<IndexColumn> columns) {
- GridCacheContext<?, ?> cctx = tbl.rowDescriptor().context();
-
- if (cctx != null && cctx.config().getQueryParallelism() > 1)
- return new GridH2TreeIndex(idxName, tbl, pk, columns, cctx.config().getQueryParallelism());
+ final int segments = tbl.rowDescriptor().context().config().getQueryParallelism();
- return new GridH2TreeIndex(idxName, tbl, pk, columns, 1);
+ return new GridH2TreeIndex(idxName, tbl, pk, columns, segments);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 131e03b..89d661d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -183,7 +183,15 @@ public abstract class GridH2IndexBase extends BaseIndex {
* @return Index segment ID for current query context.
*/
protected int threadLocalSegment() {
- return 0;
+ GridH2QueryContext qctx = GridH2QueryContext.get();
+
+ if(segmentsCount() == 1)
+ return 0;
+
+ if(qctx == null)
+ throw new IllegalStateException("GridH2QueryContext is not initialized.");
+
+ return qctx.segment();
}
/**
@@ -669,7 +677,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
throw new GridH2RetryException("Failed to find node.");
}
- return new SegmentKey(node, segment(partition));
+ return new SegmentKey(node, segmentForPartition(partition));
}
/** */
@@ -829,19 +837,16 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
/** @return Index segments count. */
- protected int segmentsCount() {
- return 1;
- }
+ protected abstract int segmentsCount();
/**
* @param partition Partition idx.
* @return Segment ID for given key
*/
- protected int segment(int partition) {
- return 0;
+ protected int segmentForPartition(int partition){
+ return segmentsCount() == 1 ? 0 : (partition % segmentsCount());
}
-
/**
* Simple cursor from a single node.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 0829df0..64ca9ea 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -50,7 +50,6 @@ import org.jetbrains.annotations.Nullable;
*/
@SuppressWarnings("ComparatorNotSerializable")
public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridSearchRowPointer> {
-
/** */
private static Field KEY_FIELD;
@@ -224,20 +223,6 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
super.destroy();
}
-
- /** {@inheritDoc} */
- protected int threadLocalSegment() {
- GridH2QueryContext qctx = GridH2QueryContext.get();
-
- if(segments.length == 1)
- return 0;
-
- if(qctx == null)
- throw new IllegalStateException("GridH2QueryContext is not initialized.");
-
- return qctx.segment();
- }
-
/** {@inheritDoc} */
@Override public long getRowCount(@Nullable Session ses) {
IndexingQueryFilter f = threadLocalFilter();
@@ -433,7 +418,7 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
/** {@inheritDoc} */
@Override public GridH2Row put(GridH2Row row) {
- int seg = segment(row);
+ int seg = segmentForRow(row);
return segments[seg].put(row, row);
}
@@ -442,7 +427,7 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
@Override public GridH2Row remove(SearchRow row) {
GridSearchRowPointer comparable = comparable(row, 0);
- int seg = segment(row);
+ int seg = segmentForRow(row);
return segments[seg].remove(comparable);
}
@@ -453,18 +438,10 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
}
/**
- * @param partition Parttition idx.
- * @return index currentSegment Id for given key
- */
- protected int segment(int partition) {
- return partition % segments.length;
- }
-
- /**
- * @param row
- * @return index currentSegment Id for given row
+ * @param row Table row.
+ * @return Segment ID for given row.
*/
- private int segment(SearchRow row) {
+ private int segmentForRow(SearchRow row) {
assert row != null;
CacheObject key;
@@ -491,7 +468,7 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
else
key = ctx.toCacheKeyObject(row.getValue(0));
- return segment(ctx.affinity().partition(key));
+ return segmentForPartition(ctx.affinity().partition(key));
}
else
return 0;
[17/50] [abbrv] ignite git commit: Minor
Posted by av...@apache.org.
Minor
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f684c00a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f684c00a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f684c00a
Branch: refs/heads/master
Commit: f684c00affa25d67557b382e6ffaf2d1de45c964
Parents: e69d3c3
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 12:00:41 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 12:00:41 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/query/h2/IgniteH2Indexing.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f684c00a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 652d1f3..de48eb9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2831,7 +2831,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
if (!ctor.isAccessible())
ctor.setAccessible(true);
- final int segments = tbl.rowDescriptor().context().config().getQueryParallelism();
+ final int segments = tbl.rowDescriptor().configuration().getQueryParallelism();
return (SpatialIndex)ctor.newInstance(tbl, idxName, segments, cols);
}
@@ -2848,7 +2848,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @return
*/
private Index createTreeIndex(String idxName, GridH2Table tbl, boolean pk, List<IndexColumn> columns) {
- final int segments = tbl.rowDescriptor().context().config().getQueryParallelism();
+ final int segments = tbl.rowDescriptor().configuration().getQueryParallelism();
return new GridH2TreeIndex(idxName, tbl, pk, columns, segments);
}
[43/50] [abbrv] ignite git commit: ignite-1.9 - compilation
Posted by av...@apache.org.
ignite-1.9 - compilation
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/cf3de19c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/cf3de19c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/cf3de19c
Branch: refs/heads/master
Commit: cf3de19ccacf5c15c16df8a4e4c193d88c7dc56b
Parents: e3d7326
Author: Sergi Vladykin <se...@gmail.com>
Authored: Wed Feb 22 16:43:05 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Wed Feb 22 16:43:05 2017 +0300
----------------------------------------------------------------------
.../query/h2/GridH2IndexingGeoSelfTest.java | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cf3de19c/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
index 839514b..b53387f 100644
--- a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
+++ b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingGeoSelfTest.java
@@ -43,7 +43,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.GridTestUtils;
-import org.jetbrains.annotations.NotNull;
/**
* Geo-indexing test.
@@ -90,7 +89,8 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
* @throws Exception If failed.
*/
public void testPrimitiveGeometry() throws Exception {
- IgniteCache<Long, Geometry> cache = grid(0).getOrCreateCache(cacheConfig("geom", true, Long.class, Geometry.class));
+ IgniteCache<Long, Geometry> cache = grid(0).getOrCreateCache(
+ this.<Long, Geometry>cacheConfig("geom", true, Long.class, Geometry.class));
try {
WKTReader r = new WKTReader();
@@ -114,7 +114,7 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
@SuppressWarnings("unchecked")
public void testGeo() throws Exception {
IgniteCache<Integer, EnemyCamp> cache = grid(0).getOrCreateCache(
- cacheConfig("camp", true, Integer.class, EnemyCamp.class));
+ this.<Integer, EnemyCamp>cacheConfig("camp", true, Integer.class, EnemyCamp.class));
try {
WKTReader r = new WKTReader();
@@ -301,8 +301,10 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
* @throws Exception if fails.
*/
public void testSegmentedGeoIndexJoin() throws Exception {
- IgniteCache<Integer, Enemy> c1 = ignite(0).getOrCreateCache(cacheConfig("enemy", true, Integer.class, Enemy.class));
- IgniteCache<Integer, EnemyCamp> c2 = ignite(0).getOrCreateCache(cacheConfig("camp", true, Integer.class, EnemyCamp.class));
+ IgniteCache<Integer, Enemy> c1 = ignite(0).getOrCreateCache(
+ this.<Integer, Enemy>cacheConfig("enemy", true, Integer.class, Enemy.class));
+ IgniteCache<Integer, EnemyCamp> c2 = ignite(0).getOrCreateCache(
+ this.<Integer, EnemyCamp>cacheConfig("camp", true, Integer.class, EnemyCamp.class));
try {
fillCache();
@@ -321,8 +323,10 @@ public class GridH2IndexingGeoSelfTest extends GridCacheAbstractSelfTest {
* @throws Exception if fails.
*/
public void testSegmentedGeoIndexJoin2() throws Exception {
- IgniteCache<Integer, Enemy> c1 = ignite(0).getOrCreateCache(cacheConfig("enemy", true, Integer.class, Enemy.class));
- IgniteCache<Integer, EnemyCamp> c2 = ignite(0).getOrCreateCache(cacheConfig("camp", false, Integer.class, EnemyCamp.class));
+ IgniteCache<Integer, Enemy> c1 = ignite(0).getOrCreateCache(
+ this.<Integer, Enemy>cacheConfig("enemy", true, Integer.class, Enemy.class));
+ IgniteCache<Integer, EnemyCamp> c2 = ignite(0).getOrCreateCache(
+ this.<Integer, EnemyCamp>cacheConfig("camp", false, Integer.class, EnemyCamp.class));
try {
fillCache();
[18/50] [abbrv] ignite git commit: Merge remote-tracking branch
'apache/ignite-1.9' into ignite-4106-1.9
Posted by av...@apache.org.
Merge remote-tracking branch 'apache/ignite-1.9' into ignite-4106-1.9
# Conflicts:
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
# modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b774f6d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b774f6d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b774f6d
Branch: refs/heads/master
Commit: 5b774f6d9dc7faa60a95b1206d30115b0ef67d16
Parents: f684c00 166e65c
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 12:10:38 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 12:10:38 2017 +0300
----------------------------------------------------------------------
.../examples/java8/spark/SharedRDDExample.java | 4 +-
.../jdbc2/JdbcAbstractDmlStatementSelfTest.java | 49 +-
.../jdbc2/JdbcInsertStatementSelfTest.java | 51 +
.../jdbc2/JdbcMergeStatementSelfTest.java | 51 +
.../internal/jdbc2/JdbcStreamingSelfTest.java | 189 ++
.../jdbc2/JdbcUpdateStatementSelfTest.java | 50 +
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 1 +
.../org/apache/ignite/IgniteJdbcDriver.java | 30 +
.../apache/ignite/IgniteSystemProperties.java | 6 +
.../ignite/internal/jdbc2/JdbcConnection.java | 72 +-
.../internal/jdbc2/JdbcPreparedStatement.java | 34 +-
.../ignite/internal/jdbc2/JdbcStatement.java | 20 +-
.../jdbc2/JdbcStreamedPreparedStatement.java | 59 +
.../cache/query/GridCacheSqlQuery.java | 45 +-
.../processors/query/GridQueryIndexing.java | 35 +
.../processors/query/GridQueryProcessor.java | 63 +-
.../visor/cache/VisorCacheClearTask.java | 88 +-
.../visor/compute/VisorGatewayTask.java | 30 +-
.../junits/multijvm/IgniteProcessProxy.java | 5 +-
.../query/h2/DmlStatementsProcessor.java | 267 ++-
.../processors/query/h2/IgniteH2Indexing.java | 160 +-
.../query/h2/dml/UpdatePlanBuilder.java | 2 +-
.../query/h2/opt/GridH2CollocationModel.java | 78 +-
.../query/h2/opt/GridH2IndexBase.java | 47 +-
.../processors/query/h2/sql/DmlAstUtils.java | 46 +-
.../processors/query/h2/sql/GridSqlAlias.java | 13 +-
.../processors/query/h2/sql/GridSqlArray.java | 8 +-
.../processors/query/h2/sql/GridSqlAst.java | 61 +
.../processors/query/h2/sql/GridSqlColumn.java | 85 +-
.../processors/query/h2/sql/GridSqlConst.java | 6 +-
.../processors/query/h2/sql/GridSqlElement.java | 43 +-
.../query/h2/sql/GridSqlFunction.java | 16 +-
.../processors/query/h2/sql/GridSqlJoin.java | 35 +-
.../processors/query/h2/sql/GridSqlKeyword.java | 3 +-
.../query/h2/sql/GridSqlOperation.java | 6 +-
.../query/h2/sql/GridSqlOperationType.java | 4 +-
.../query/h2/sql/GridSqlParameter.java | 4 +-
.../query/h2/sql/GridSqlPlaceholder.java | 2 +-
.../processors/query/h2/sql/GridSqlQuery.java | 80 +-
.../query/h2/sql/GridSqlQueryParser.java | 228 ++-
.../query/h2/sql/GridSqlQuerySplitter.java | 1634 +++++++++++++++---
.../processors/query/h2/sql/GridSqlSelect.java | 121 +-
.../query/h2/sql/GridSqlStatement.java | 6 +-
.../query/h2/sql/GridSqlSubquery.java | 31 +-
.../processors/query/h2/sql/GridSqlTable.java | 19 +-
.../processors/query/h2/sql/GridSqlType.java | 6 +-
.../processors/query/h2/sql/GridSqlUnion.java | 66 +-
.../processors/query/h2/sql/GridSqlValue.java | 25 -
.../query/h2/twostep/GridMapQueryExecutor.java | 6 +-
.../query/h2/twostep/GridMergeIndex.java | 418 ++++-
.../query/h2/twostep/GridMergeIndexSorted.java | 284 +++
.../h2/twostep/GridMergeIndexUnsorted.java | 40 +-
.../h2/twostep/GridReduceQueryExecutor.java | 56 +-
.../h2/twostep/msg/GridH2QueryRequest.java | 9 +-
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 2 +-
...niteCacheAbstractInsertSqlQuerySelfTest.java | 6 +-
.../IgniteCacheInsertSqlQuerySelfTest.java | 14 +
.../query/IgniteSqlSplitterSelfTest.java | 100 +-
.../query/h2/sql/GridQueryParsingTest.java | 41 +-
modules/spark-2.10/pom.xml | 54 +
modules/spark/pom.xml | 183 +-
.../org/apache/ignite/spark/IgniteContext.scala | 22 +-
.../spark/JavaEmbeddedIgniteRDDSelfTest.java | 10 +-
.../spark/JavaStandaloneIgniteRDDSelfTest.java | 22 +-
.../web-console/backend/services/activities.js | 7 +-
.../list-of-registered-users.controller.js | 14 +-
.../configuration/configuration.module.js | 6 +-
.../configuration/summary/summary.worker.js | 28 +-
.../console/agent/handlers/RestListener.java | 9 +-
modules/yardstick/DEVNOTES-standalone.txt | 5 +-
modules/yardstick/README.txt | 85 +-
.../config/benchmark-atomic-win.properties | 2 +-
.../config/benchmark-atomic.properties | 35 +-
.../config/benchmark-bin-identity.properties | 16 +-
.../config/benchmark-cache-load-win.properties | 2 +-
.../config/benchmark-cache-load.properties | 4 +-
.../config/benchmark-client-mode.properties | 68 +-
.../config/benchmark-compute-win.properties | 2 +-
.../config/benchmark-compute.properties | 30 +-
.../config/benchmark-failover.properties | 2 +-
.../yardstick/config/benchmark-full.properties | 62 +-
.../config/benchmark-multicast.properties | 107 +-
.../config/benchmark-put-indexed-val.properties | 23 +-
.../benchmark-query-put-separated.properties | 3 +-
.../config/benchmark-query-win.properties | 2 +-
.../yardstick/config/benchmark-query.properties | 33 +-
.../config/benchmark-remote-sample.properties | 80 +
.../config/benchmark-remote.properties | 119 ++
.../config/benchmark-sample.properties | 22 +-
.../config/benchmark-sql-dml.properties | 36 +-
.../yardstick/config/benchmark-store.properties | 3 +-
.../config/benchmark-tx-win.properties | 2 +-
.../yardstick/config/benchmark-tx.properties | 33 +-
.../yardstick/config/benchmark-win.properties | 2 +-
modules/yardstick/config/benchmark.properties | 76 +-
.../yardstick/config/ignite-remote-config.xml | 47 +
.../test-max-int-values-offheap.properties | 3 +-
.../test-max-int-values-onheap.properties | 3 +-
.../config/test-max-int-values-swap.properties | 3 +-
modules/yardstick/pom-standalone.xml | 2 +-
modules/yardstick/pom.xml | 2 +-
parent/pom.xml | 3 +-
pom.xml | 5 +-
103 files changed, 4874 insertions(+), 1363 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index 37f0ade,2abb3a9..5d2b728
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@@ -84,26 -85,48 +85,39 @@@ public interface GridQueryIndexing
/**
* Queries individual fields (generally used by JDBC drivers).
*
- * @param spaceName Space name.
+ * @param cctx Cache context.
* @param qry Query.
- * @param params Query parameters.
* @param filter Space name and key filter.
- * @param enforceJoinOrder Enforce join order of tables in the query.
- * @param timeout Query timeout in milliseconds.
* @param cancel Query cancel.
- * @return Query result.
- * @throws IgniteCheckedException If failed.
+ * @return Cursor.
*/
- public GridQueryFieldsResult queryLocalSqlFields(@Nullable String spaceName, String qry,
- Collection<Object> params, IndexingQueryFilter filter, boolean enforceJoinOrder, int timeout,
- GridQueryCancel cancel) throws IgniteCheckedException;
+ public <K, V> QueryCursor<List<?>> queryLocalSqlFields(GridCacheContext<?, ?> cctx, SqlFieldsQuery qry,
+ IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException;
/**
+ * Perform a MERGE statement using data streamer as receiver.
+ *
+ * @param spaceName Space name.
+ * @param qry Query.
+ * @param params Query parameters.
+ * @param streamer Data streamer to feed data to.
+ * @return Query result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long streamUpdateQuery(@Nullable final String spaceName, final String qry,
+ @Nullable final Object[] params, IgniteDataStreamer<?, ?> streamer) throws IgniteCheckedException;
+
+ /**
* Executes regular query.
*
- * @param spaceName Space name.
+ * @param cctx Cache context.
* @param qry Query.
- * @param alias Table alias used in Query.
- * @param params Query parameters.
- * @param type Query return type.
* @param filter Space name and key filter.
- * @return Queried rows.
- * @throws IgniteCheckedException If failed.
+ * @param keepBinary Keep binary flag.
+ * @return Cursor.
*/
- public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName, String qry,
- String alias, Collection<Object> params, GridQueryTypeDescriptor type, IndexingQueryFilter filter)
- throws IgniteCheckedException;
+ public <K, V> QueryCursor<Cache.Entry<K,V>> queryLocalSql(GridCacheContext<?, ?> cctx, SqlQuery qry,
+ IndexingQueryFilter filter, boolean keepBinary) throws IgniteCheckedException;
/**
* Executes text query.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index de48eb9,15e7fc6..057fd10
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@@ -93,8 -91,8 +94,9 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.query.GridQueryIndexing;
import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+ import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2DefaultTableEngine;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOffheap;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap;
@@@ -1352,9 -1281,7 +1400,9 @@@ public class IgniteH2Indexing implement
final boolean distributedJoins = qry.isDistributedJoins() && cctx.isPartitioned();
final boolean grpByCollocated = qry.isCollocated();
+ final DistributedJoinMode distributedJoinMode = distributedJoinMode(qry.isLocal(), distributedJoins);
+
- GridCacheTwoStepQuery twoStepQry;
+ GridCacheTwoStepQuery twoStepQry = null;
List<GridQueryFieldMetadata> meta;
final TwoStepCachedQueryKey cachedQryKey = new TwoStepCachedQueryKey(space, sqlQry, grpByCollocated,
@@@ -1368,12 -1295,13 +1416,13 @@@
else {
final UUID locNodeId = ctx.localNodeId();
- setupConnection(c, distributedJoins, enforceJoinOrder);
+ // Here we will just parse the statement, no need to optimize it at all.
+ setupConnection(c, /*distributedJoins*/false, /*enforceJoinOrder*/true);
GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, 0, PREPARE)
- .distributedJoins(distributedJoins));
+ .distributedJoinMode(distributedJoinMode));
- PreparedStatement stmt;
+ PreparedStatement stmt = null;
boolean cachesCreated = false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 89d661d,bdfddd5..31057c7
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@@ -262,14 -245,15 +262,15 @@@ public abstract class GridH2IndexBase e
* @param filter Current filter.
* @return Multiplier.
*/
- public int getDistributedMultiplier(Session ses, TableFilter[] filters, int filter) {
+ public final int getDistributedMultiplier(Session ses, TableFilter[] filters, int filter) {
GridH2QueryContext qctx = GridH2QueryContext.get();
- // We do complex optimizations with respect to distributed joins only on prepare stage
- // because on run stage reordering of joined tables by Optimizer is explicitly disabled
- // and thus multiplier will be always the same, so it will not affect choice of index.
+ // We do optimizations with respect to distributed joins only on PREPARE stage only.
+ // Notice that we check for isJoinBatchEnabled, because we can do multiple different
+ // optimization passes on PREPARE stage.
// Query expressions can not be distributed as well.
- if (qctx == null || qctx.type() != PREPARE || qctx.distributedJoinMode() == OFF || ses.isPreparingQueryExpression())
- if (qctx == null || qctx.type() != PREPARE || !qctx.distributedJoins() ||
++ if (qctx == null || qctx.type() != PREPARE || qctx.distributedJoinMode() == OFF ||
+ !ses.isJoinBatchEnabled() || ses.isPreparingQueryExpression())
return GridH2CollocationModel.MULTIPLIER_COLLOCATED;
// We have to clear this cache because normally sub-query plan cost does not depend on anything
@@@ -391,18 -375,16 +392,18 @@@
if (affCol != null) {
affColId = affCol.column.getColumnId();
int[] masks = filter.getMasks();
- ucast = masks != null && masks[affColId] == IndexCondition.EQUALITY;
- }
- else {
- affColId = -1;
- ucast = false;
+
+ if (masks != null) {
+ ucast = (masks[affColId] & IndexCondition.EQUALITY) != 0 ||
+ (masks[KEY_COL] & IndexCondition.EQUALITY) != 0;
+ }
}
- GridCacheContext<?,?> cctx = getTable().rowDescriptor().context();
+ GridCacheContext<?, ?> cctx = getTable().rowDescriptor().context();
- return new DistributedLookupBatch(cctx, ucast, affColId);
+ boolean isLocal = qctx.distributedJoinMode() == LOCAL_ONLY;
+
+ return new DistributedLookupBatch(cctx, ucast, affColId, isLocal);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 5027c9a,b0fa639..72a34a6
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@@ -86,8 -84,7 +86,9 @@@ import static org.apache.ignite.events.
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+ import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.setupConnection;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.MAP;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REPLICATED;
import static org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.QUERY_POOL;
@@@ -433,7 -435,8 +434,8 @@@ public class GridMapQueryExecutor
req.partitions(),
null,
req.pageSize(),
- false,
+ OFF,
+ true,
req.timeout());
}
@@@ -491,7 -457,8 +493,8 @@@
parts,
req.tables(),
req.pageSize(),
- req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS),
+ joinMode,
+ req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER),
req.timeout());
}
@@@ -519,7 -484,8 +522,8 @@@
int[] parts,
Collection<String> tbls,
int pageSize,
- boolean distributedJoins,
+ DistributedJoinMode distributedJoinMode,
+ boolean enforceJoinOrder,
int timeout
) {
// Prepare to run queries.
@@@ -580,8 -545,7 +584,8 @@@
Connection conn = h2.connectionForSpace(mainCctx.name());
- setupConnection(conn, distributedJoins, enforceJoinOrder);
+ // Here we enforce join order to have the same behavior on all the nodes.
- h2.setupConnection(conn, distributedJoinMode != OFF, true);
++ h2.setupConnection(conn, distributedJoinMode != OFF, enforceJoinOrder);
GridH2QueryContext.set(qctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 8837046,f813cae..3cfaae9
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@@ -62,12 -62,11 +62,11 @@@ import org.apache.ignite.internal.proce
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
-import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext;
- import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlType;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse;
@@@ -100,8 -99,9 +99,10 @@@ import org.jsr166.ConcurrentHashMap8
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
+ import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.setupConnection;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
+ import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter.mergeTableIdentifier;
/**
* Reduce query executor.
@@@ -616,6 -610,12 +618,15 @@@ public class GridReduceQueryExecutor
if (oldStyle && distributedJoins)
throw new CacheException("Failed to enable distributed joins. Topology contains older data nodes.");
+ // Always enforce join order on map side to have consistent behavior.
+ int flags = GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER;
+
+ if (distributedJoins)
+ flags |= GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS;
+
++ if(qry.isLocal())
++ flags |= GridH2QueryRequest.FLAG_IS_LOCAL;
++
if (send(nodes,
oldStyle ?
new GridQueryRequest(qryReqId,
@@@ -634,13 -634,14 +645,12 @@@
.tables(distributedJoins ? qry.tables() : null)
.partitions(convert(partsMap))
.queries(mapQrys)
- .flags((qry.isLocal() ? GridH2QueryRequest.FLAG_IS_LOCAL : 0) |
- (distributedJoins ? GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS : 0))
+ .flags(flags)
.timeout(timeoutMillis),
oldStyle && partsMap != null ? new ExplicitPartitionsSpecializer(partsMap) : null,
- distributedJoins)
- ) {
- awaitAllReplies(r, nodes);
+ false)) {
- cancel.checkCancelled();
+ awaitAllReplies(r, nodes, cancel);
Object state = r.state.get();
@@@ -700,10 -700,10 +709,10 @@@
UUID locNodeId = ctx.localNodeId();
- h2.setupConnection(r.conn, false, enforceJoinOrder);
+ setupConnection(r.conn, false, enforceJoinOrder);
GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, qryReqId, REDUCE)
- .pageSize(r.pageSize).distributedJoins(false));
+ .pageSize(r.pageSize).distributedJoinMode(OFF));
try {
if (qry.explain())
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
index ec49aff,e5dbf33..0ad534c
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
@@@ -51,10 -51,10 +51,15 @@@ public class GridH2QueryRequest impleme
public static int FLAG_DISTRIBUTED_JOINS = 1;
/**
+ * Remote map query executor will enforce join order for the received map queries.
+ */
+ public static int FLAG_ENFORCE_JOIN_ORDER = 1 << 1;
+
++ /**
+ * Restrict distributed joins range-requests to local index segments. Range requests to other nodes will not be sent.
+ */
- public static int FLAG_IS_LOCAL = 2;
++ public static int FLAG_IS_LOCAL = 1 << 2;
+
/** */
private long reqId;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5b774f6d/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
----------------------------------------------------------------------
diff --cc modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 4ae2f91,432ed34..37dea47
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@@ -34,9 -34,8 +34,10 @@@ import org.apache.ignite.cache.CacheKey
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.affinity.Affinity;
+ import org.apache.ignite.cache.affinity.AffinityKeyMapped;
import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
+import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.cluster.ClusterNode;
[23/50] [abbrv] ignite git commit: Removed redundant method
Posted by av...@apache.org.
Removed redundant method
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/990f64d7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/990f64d7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/990f64d7
Branch: refs/heads/master
Commit: 990f64d7694204d2e7889d9aad2156343988dbad
Parents: 14290d3
Author: Alexander Paschenko <al...@gmail.com>
Authored: Tue Feb 21 15:28:46 2017 +0300
Committer: Alexander Paschenko <al...@gmail.com>
Committed: Tue Feb 21 15:28:46 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/query/h2/IgniteH2Indexing.java | 5 -----
.../internal/processors/query/h2/opt/GridH2RowDescriptor.java | 5 -----
2 files changed, 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/990f64d7/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 15e7fc6..9a355e5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -3221,11 +3221,6 @@ public class IgniteH2Indexing implements GridQueryIndexing {
@Override public boolean snapshotableIndex() {
return snapshotableIdx;
}
-
- /** {@inheritDoc} */
- @Override public boolean quoteAllIdentifiers() {
- return schema.escapeAll();
- }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/990f64d7/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2RowDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2RowDescriptor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2RowDescriptor.java
index 3465ed7..8136ee1 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2RowDescriptor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2RowDescriptor.java
@@ -163,9 +163,4 @@ public interface GridH2RowDescriptor extends GridOffHeapSmartPointerFactory<Grid
* @return {@code True} if index should support snapshots.
*/
public boolean snapshotableIndex();
-
- /**
- * @return Escape all identifiers.
- */
- public boolean quoteAllIdentifiers();
}
\ No newline at end of file
[26/50] [abbrv] ignite git commit: Merge branch 'ignite-4732' of
https://github.com/gridgain/apache-ignite into ignite-1.9
Posted by av...@apache.org.
Merge branch 'ignite-4732' of https://github.com/gridgain/apache-ignite into ignite-1.9
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/207a669b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/207a669b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/207a669b
Branch: refs/heads/master
Commit: 207a669b7b9b48834791bddd8fe28d4549d52de8
Parents: a65889a f0ef842
Author: Sergi Vladykin <se...@gmail.com>
Authored: Tue Feb 21 16:18:30 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Tue Feb 21 16:18:30 2017 +0300
----------------------------------------------------------------------
.../internal/processors/query/h2/IgniteH2Indexing.java | 5 -----
.../processors/query/h2/opt/GridH2RowDescriptor.java | 5 -----
.../ignite/internal/processors/query/h2/sql/DmlAstUtils.java | 8 ++++----
.../internal/processors/query/h2/sql/GridSqlFunction.java | 2 +-
4 files changed, 5 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
[35/50] [abbrv] ignite git commit: ignite-3860 - parsing test
Posted by av...@apache.org.
ignite-3860 - parsing test
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/963a9d9f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/963a9d9f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/963a9d9f
Branch: refs/heads/master
Commit: 963a9d9fe39a67097c848c7b232ddb220ea34189
Parents: 43fea35
Author: Sergi Vladykin <se...@gmail.com>
Authored: Wed Feb 22 12:38:10 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Wed Feb 22 12:38:10 2017 +0300
----------------------------------------------------------------------
.../internal/processors/query/h2/sql/GridQueryParsingTest.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/963a9d9f/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index b869da9..225c838 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -301,6 +301,9 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
// Must be distinct objects, even if it is the same table.
assertNotSame(tbl1, tbl2);
+ assertNotNull(tbl1.dataTable());
+ assertNotNull(tbl2.dataTable());
+ assertSame(tbl1.dataTable(), tbl2.dataTable());
GridSqlColumn col1 = (GridSqlColumn)select.column(0);
GridSqlColumn col2 = (GridSqlColumn)select.column(1);
[19/50] [abbrv] ignite git commit: Minor
Posted by av...@apache.org.
Minor
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/56ce2237
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/56ce2237
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/56ce2237
Branch: refs/heads/master
Commit: 56ce2237a87c2285d9e2f2e8ce433ba17d27c64c
Parents: 5b774f6
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 12:23:28 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 12:52:33 2017 +0300
----------------------------------------------------------------------
.../processors/query/h2/twostep/GridMapQueryExecutor.java | 7 +++++--
.../processors/query/h2/twostep/GridMergeIndexSorted.java | 4 ++--
.../processors/query/h2/twostep/GridReduceQueryExecutor.java | 2 +-
3 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/56ce2237/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 72a34a6..f002a5e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -458,6 +458,8 @@ public class GridMapQueryExecutor {
req.isFlagSet(GridH2QueryRequest.FLAG_IS_LOCAL),
req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS));
+ final boolean enforceJoinOrder = req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER);
+
for (int i = 1; i < mainCctx.config().getQueryParallelism(); i++) {
final int segment = i;
@@ -475,6 +477,7 @@ public class GridMapQueryExecutor {
req.tables(),
req.pageSize(),
joinMode,
+ enforceJoinOrder,
req.timeout());
return null;
@@ -494,7 +497,7 @@ public class GridMapQueryExecutor {
req.tables(),
req.pageSize(),
joinMode,
- req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER),
+ enforceJoinOrder,
req.timeout());
}
@@ -585,7 +588,7 @@ public class GridMapQueryExecutor {
Connection conn = h2.connectionForSpace(mainCctx.name());
// Here we enforce join order to have the same behavior on all the nodes.
- h2.setupConnection(conn, distributedJoinMode != OFF, enforceJoinOrder);
+ setupConnection(conn, distributedJoinMode != OFF, enforceJoinOrder);
GridH2QueryContext.set(qctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/56ce2237/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
index a1b6691..32c676d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
@@ -85,8 +85,8 @@ public final class GridMergeIndexSorted extends GridMergeIndex {
}
/** {@inheritDoc} */
- @Override public void setSources(Collection<ClusterNode> nodes) {
- super.setSources(nodes);
+ @Override public void setSources(Collection<ClusterNode> nodes, int segmentsCnt) {
+ super.setSources(nodes, segmentsCnt);
streamsMap = U.newHashMap(nodes.size());
streams = new RowStream[nodes.size()];
http://git-wip-us.apache.org/repos/asf/ignite/blob/56ce2237/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 3cfaae9..4cae6ac 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -624,7 +624,7 @@ public class GridReduceQueryExecutor {
if (distributedJoins)
flags |= GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS;
- if(qry.isLocal())
+ if (qry.isLocal())
flags |= GridH2QueryRequest.FLAG_IS_LOCAL;
if (send(nodes,
[36/50] [abbrv] ignite git commit: Merge branch 'ignite-4106-1.9' of
https://github.com/gridgain/apache-ignite into ignite-1.9
Posted by av...@apache.org.
Merge branch 'ignite-4106-1.9' of https://github.com/gridgain/apache-ignite into ignite-1.9
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/658b4ad5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/658b4ad5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/658b4ad5
Branch: refs/heads/master
Commit: 658b4ad5a3bf17fcb3fb9f364e47b4e28f5eea64
Parents: 963a9d9 4dc3694
Author: Sergi Vladykin <se...@gmail.com>
Authored: Wed Feb 22 13:17:43 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Wed Feb 22 13:17:43 2017 +0300
----------------------------------------------------------------------
.../configuration/CacheConfiguration.java | 48 +++
.../processors/cache/GridCacheProcessor.java | 3 +
.../processors/cache/IgniteCacheProxy.java | 6 +-
.../closure/GridClosureProcessor.java | 2 +-
.../processors/query/GridQueryIndexing.java | 27 +-
.../processors/query/GridQueryProcessor.java | 141 ++-----
.../messages/GridQueryNextPageRequest.java | 29 +-
.../messages/GridQueryNextPageResponse.java | 29 +-
.../query/h2/opt/GridH2SpatialIndex.java | 56 ++-
.../query/h2/GridH2IndexingGeoSelfTest.java | 405 ++++++++++++++-----
.../h2/GridH2IndexingSegmentedGeoSelfTest.java | 37 ++
.../testsuites/GeoSpatialIndexingTestSuite.java | 2 +
.../cache/query/GridCacheTwoStepQuery.java | 17 +
.../processors/query/h2/IgniteH2Indexing.java | 238 +++++++++--
.../query/h2/opt/DistributedJoinMode.java | 51 +++
.../query/h2/opt/GridH2IndexBase.java | 303 ++++++++++----
.../query/h2/opt/GridH2QueryContext.java | 84 +++-
.../query/h2/opt/GridH2TreeIndex.java | 151 ++++---
.../query/h2/sql/GridSqlQuerySplitter.java | 2 +-
.../query/h2/twostep/GridMapQueryExecutor.java | 233 ++++++++---
.../query/h2/twostep/GridMergeIndex.java | 39 +-
.../query/h2/twostep/GridMergeIndexSorted.java | 4 +-
.../h2/twostep/GridReduceQueryExecutor.java | 69 ++--
.../h2/twostep/msg/GridH2IndexRangeRequest.java | 60 ++-
.../twostep/msg/GridH2IndexRangeResponse.java | 62 ++-
.../h2/twostep/msg/GridH2QueryRequest.java | 5 +
.../query/IgniteSqlSegmentedIndexSelfTest.java | 263 ++++++++++++
.../query/IgniteSqlSplitterSelfTest.java | 136 ++++++-
.../h2/GridIndexingSpiAbstractSelfTest.java | 26 +-
.../FetchingQueryCursorStressTest.java | 277 +++++++++++++
.../IgniteCacheQuerySelfTestSuite.java | 2 +
31 files changed, 2262 insertions(+), 545 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/658b4ad5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/658b4ad5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
[20/50] [abbrv] ignite git commit: IGNITE-3710 Upgrade ignite-spark
module to Spark 2.0 (cherry picked from commit a71a230)
Posted by av...@apache.org.
IGNITE-3710 Upgrade ignite-spark module to Spark 2.0
(cherry picked from commit a71a230)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a65889a7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a65889a7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a65889a7
Branch: refs/heads/master
Commit: a65889a75b0ba7b3fe74d97df82e2214ff1d9b52
Parents: 166e65c
Author: Evgenii Zhuravlev <ez...@gridgain.com>
Authored: Tue Feb 21 13:25:41 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Tue Feb 21 13:27:55 2017 +0300
----------------------------------------------------------------------
examples/pom-standalone-lgpl.xml | 6 ++++++
examples/pom-standalone.xml | 6 ++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a65889a7/examples/pom-standalone-lgpl.xml
----------------------------------------------------------------------
diff --git a/examples/pom-standalone-lgpl.xml b/examples/pom-standalone-lgpl.xml
index 3814672..c09cc6b 100644
--- a/examples/pom-standalone-lgpl.xml
+++ b/examples/pom-standalone-lgpl.xml
@@ -106,6 +106,12 @@
<artifactId>ignite-scalar</artifactId>
<version>to_be_replaced_by_ignite_version</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-spark</artifactId>
+ <version>to_be_replaced_by_ignite_version</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a65889a7/examples/pom-standalone.xml
----------------------------------------------------------------------
diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml
index 08c27fe..b3843e6 100644
--- a/examples/pom-standalone.xml
+++ b/examples/pom-standalone.xml
@@ -106,6 +106,12 @@
<artifactId>ignite-scalar</artifactId>
<version>to_be_replaced_by_ignite_version</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-spark</artifactId>
+ <version>to_be_replaced_by_ignite_version</version>
+ </dependency>
</dependencies>
<build>
[30/50] [abbrv] ignite git commit: IGNITE-4410 .NET: Move
default-config-dotnet.xml to fabric-base so it is not included in Hadoop
releases
Posted by av...@apache.org.
IGNITE-4410 .NET: Move default-config-dotnet.xml to fabric-base so it is not included in Hadoop releases
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a1601625
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a1601625
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a1601625
Branch: refs/heads/master
Commit: a16016257c6e8ea0e33ed1b4cfdf0bcc655a16c4
Parents: bf337bc
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Tue Feb 21 19:39:22 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Tue Feb 21 19:39:22 2017 +0300
----------------------------------------------------------------------
assembly/release-base.xml | 5 -----
assembly/release-fabric-base.xml | 5 +++++
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a1601625/assembly/release-base.xml
----------------------------------------------------------------------
diff --git a/assembly/release-base.xml b/assembly/release-base.xml
index a2a9382..6870284 100644
--- a/assembly/release-base.xml
+++ b/assembly/release-base.xml
@@ -46,11 +46,6 @@
<source>config/java.util.logging.properties</source>
<outputDirectory>/config</outputDirectory>
</file>
-
- <file>
- <source>config/dotnet/default-dotnet.xml</source>
- <outputDirectory>/config/dotnet</outputDirectory>
- </file>
</files>
<fileSets>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a1601625/assembly/release-fabric-base.xml
----------------------------------------------------------------------
diff --git a/assembly/release-fabric-base.xml b/assembly/release-fabric-base.xml
index c008083..d7aa0a2 100644
--- a/assembly/release-fabric-base.xml
+++ b/assembly/release-fabric-base.xml
@@ -65,6 +65,11 @@
<outputDirectory>/examples/schema-import</outputDirectory>
<destName>pom.xml</destName>
</file>
+
+ <file>
+ <source>config/dotnet/default-dotnet.xml</source>
+ <outputDirectory>/config/dotnet</outputDirectory>
+ </file>
</files>
<fileSets>
[34/50] [abbrv] ignite git commit: IGNITE-4735: CPP: Fixed a problem
with missing sources having "target" word in name. This closes #1562.
Posted by av...@apache.org.
IGNITE-4735: CPP: Fixed a problem with missing sources having "target" word in name. This closes #1562.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/43fea35a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/43fea35a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/43fea35a
Branch: refs/heads/master
Commit: 43fea35aaf92fe50c74ff0359fafe5377d88637f
Parents: 437f374
Author: isapego <ig...@gmail.com>
Authored: Wed Feb 22 12:08:54 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Feb 22 12:08:54 2017 +0300
----------------------------------------------------------------------
assembly/release-sources.xml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/43fea35a/assembly/release-sources.xml
----------------------------------------------------------------------
diff --git a/assembly/release-sources.xml b/assembly/release-sources.xml
index 96e54f9..cc33f3e 100644
--- a/assembly/release-sources.xml
+++ b/assembly/release-sources.xml
@@ -31,7 +31,8 @@
<useDefaultExcludes>true</useDefaultExcludes>
<excludes>
<!-- build output -->
- <exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}.*]</exclude>
+ <exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}/.*]</exclude>
+ <exclude>%regex[(?!((?!${project.build.directory}/)[^/]+/)*src/).*${project.build.directory}]</exclude>
<!-- NOTE: Most of the following excludes should not be required
if the standard release process is followed. This is because the
[12/50] [abbrv] ignite git commit: Merge branch 'ignite-1.9' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-1.9
Posted by av...@apache.org.
Merge branch 'ignite-1.9' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-1.9
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/43bf89f9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/43bf89f9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/43bf89f9
Branch: refs/heads/master
Commit: 43bf89f97cac84fee155e5063b1b18937b74b1f0
Parents: 3737407 caf7b22
Author: Sergi Vladykin <se...@gmail.com>
Authored: Mon Feb 20 22:09:42 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Mon Feb 20 22:09:42 2017 +0300
----------------------------------------------------------------------
.../examples/java8/spark/SharedRDDExample.java | 4 +-
modules/spark-2.10/pom.xml | 54 ++++++
modules/spark/pom.xml | 183 ++++++++++++++++++-
.../org/apache/ignite/spark/IgniteContext.scala | 22 ++-
.../spark/JavaEmbeddedIgniteRDDSelfTest.java | 10 +-
.../spark/JavaStandaloneIgniteRDDSelfTest.java | 22 +--
parent/pom.xml | 3 +-
7 files changed, 270 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
[38/50] [abbrv] ignite git commit: IGNITE-4742: Fixed
IgniteCacheUpdateSqlQuerySelfTest.testTypeConversions flakiness. This closes
#1569.
Posted by av...@apache.org.
IGNITE-4742: Fixed IgniteCacheUpdateSqlQuerySelfTest.testTypeConversions flakiness. This closes #1569.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2d0d5673
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2d0d5673
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2d0d5673
Branch: refs/heads/master
Commit: 2d0d5673292814a914208770e9f3992c17a078be
Parents: 19d6c31
Author: Alexander Paschenko <al...@gmail.com>
Authored: Wed Feb 22 15:32:56 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Feb 22 15:32:56 2017 +0300
----------------------------------------------------------------------
.../cache/IgniteCacheUpdateSqlQuerySelfTest.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2d0d5673/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java
index 575f617..7ea8ba7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheUpdateSqlQuerySelfTest.java
@@ -24,6 +24,7 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.ignite.IgniteCache;
@@ -182,14 +183,14 @@ public class IgniteCacheUpdateSqlQuerySelfTest extends IgniteCacheAbstractSqlDml
cache.query(new SqlFieldsQuery("insert into \"AllTypes\"(_key, _val, \"dateCol\", \"booleanCol\"," +
"\"tsCol\") values(2, ?, '2016-11-30 12:00:00', false, DATE '2016-12-01')").setArgs(new AllTypes(2L)));
- cache.query(new SqlFieldsQuery("select \"primitiveIntsCol\" from \"AllTypes\"")).getAll();
-
- cache.query(new SqlFieldsQuery("update \"AllTypes\" set \"doubleCol\" = CAST('50' as INT)," +
- " \"booleanCol\" = 80, \"innerTypeCol\" = ?, \"strCol\" = PI(), \"shortCol\" = " +
+ List<List<?>> resLst = cache.query(new SqlFieldsQuery("update \"AllTypes\" set \"doubleCol\" = " +
+ "CAST('50' as INT), \"booleanCol\" = 80, \"innerTypeCol\" = ?, \"strCol\" = PI(), \"shortCol\" = " +
"CAST(WEEK(PARSEDATETIME('2016-11-30', 'yyyy-MM-dd')) as VARCHAR), " +
"\"sqlDateCol\"=TIMESTAMP '2016-12-02 13:47:00', \"tsCol\"=TIMESTAMPADD('MI', 2, " +
"DATEADD('DAY', 2, \"tsCol\")), \"primitiveIntsCol\" = ?, \"bytesCol\" = ?")
- .setArgs(new AllTypes.InnerType(80L), new int[] {2, 3}, new Byte[] {4, 5, 6}));
+ .setArgs(new AllTypes.InnerType(80L), new int[] {2, 3}, new Byte[] {4, 5, 6})).getAll();
+
+ assertEquals(Collections.singletonList(Collections.singletonList(1L)), resLst);
AllTypes res = (AllTypes) cache.get(2L);
[07/50] [abbrv] ignite git commit: Merge remote-tracking branch
'origin/ignite-1.9' into ignite-1.9
Posted by av...@apache.org.
Merge remote-tracking branch 'origin/ignite-1.9' into ignite-1.9
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5b94a7d5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5b94a7d5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5b94a7d5
Branch: refs/heads/master
Commit: 5b94a7d5b14600e951d4bb31b474f9fb1195f703
Parents: 32c3d6b 95dd748
Author: devozerov <vo...@gridgain.com>
Authored: Mon Feb 20 18:04:42 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Mon Feb 20 18:04:42 2017 +0300
----------------------------------------------------------------------
.../visor/cache/VisorCacheClearTask.java | 88 +++++---------------
.../visor/compute/VisorGatewayTask.java | 30 ++++++-
2 files changed, 49 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
[11/50] [abbrv] ignite git commit: ignite-3860 - merge to 1.9 #
Conflicts:
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/
Posted by av...@apache.org.
ignite-3860 - merge to 1.9
# Conflicts:
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
# modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
# modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3737407b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3737407b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3737407b
Branch: refs/heads/master
Commit: 3737407b28294e08e3e5d9a54b7efa350a7377c1
Parents: 5b94a7d
Author: Sergi Vladykin <se...@gmail.com>
Authored: Mon Feb 20 22:06:50 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Mon Feb 20 22:06:50 2017 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 6 +
.../cache/query/GridCacheSqlQuery.java | 45 +-
.../processors/query/h2/IgniteH2Indexing.java | 104 +-
.../query/h2/dml/UpdatePlanBuilder.java | 2 +-
.../query/h2/opt/GridH2CollocationModel.java | 78 +-
.../query/h2/opt/GridH2IndexBase.java | 47 +-
.../processors/query/h2/sql/DmlAstUtils.java | 46 +-
.../processors/query/h2/sql/GridSqlAlias.java | 13 +-
.../processors/query/h2/sql/GridSqlArray.java | 8 +-
.../processors/query/h2/sql/GridSqlAst.java | 61 +
.../processors/query/h2/sql/GridSqlColumn.java | 85 +-
.../processors/query/h2/sql/GridSqlConst.java | 6 +-
.../processors/query/h2/sql/GridSqlElement.java | 43 +-
.../query/h2/sql/GridSqlFunction.java | 16 +-
.../processors/query/h2/sql/GridSqlJoin.java | 35 +-
.../processors/query/h2/sql/GridSqlKeyword.java | 3 +-
.../query/h2/sql/GridSqlOperation.java | 6 +-
.../query/h2/sql/GridSqlOperationType.java | 4 +-
.../query/h2/sql/GridSqlParameter.java | 4 +-
.../query/h2/sql/GridSqlPlaceholder.java | 2 +-
.../processors/query/h2/sql/GridSqlQuery.java | 80 +-
.../query/h2/sql/GridSqlQueryParser.java | 228 ++-
.../query/h2/sql/GridSqlQuerySplitter.java | 1633 +++++++++++++++---
.../processors/query/h2/sql/GridSqlSelect.java | 121 +-
.../query/h2/sql/GridSqlStatement.java | 6 +-
.../query/h2/sql/GridSqlSubquery.java | 31 +-
.../processors/query/h2/sql/GridSqlTable.java | 19 +-
.../processors/query/h2/sql/GridSqlType.java | 6 +-
.../processors/query/h2/sql/GridSqlUnion.java | 66 +-
.../processors/query/h2/sql/GridSqlValue.java | 25 -
.../query/h2/twostep/GridMapQueryExecutor.java | 7 +-
.../query/h2/twostep/GridMergeIndex.java | 418 ++++-
.../query/h2/twostep/GridMergeIndexSorted.java | 284 +++
.../h2/twostep/GridMergeIndexUnsorted.java | 40 +-
.../h2/twostep/GridReduceQueryExecutor.java | 26 +-
.../h2/twostep/msg/GridH2QueryRequest.java | 7 +
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 2 +-
.../query/IgniteSqlSplitterSelfTest.java | 101 +-
.../query/h2/sql/GridQueryParsingTest.java | 41 +-
39 files changed, 2974 insertions(+), 781 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 914ba2f..47a66fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -403,6 +403,12 @@ public final class IgniteSystemProperties {
*/
public static final String IGNITE_SQL_MERGE_TABLE_MAX_SIZE = "IGNITE_SQL_MERGE_TABLE_MAX_SIZE";
+ /**
+ * Property controlling number of SQL result rows that will be fetched into a merge table at once before
+ * applying binary search for the bounds.
+ */
+ public static final String IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE = "IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE";
+
/** Maximum size for affinity assignment history. */
public static final String IGNITE_AFFINITY_HISTORY_SIZE = "IGNITE_AFFINITY_HISTORY_SIZE";
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
index ff1ac7f..18688b7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheSqlQuery.java
@@ -83,22 +83,11 @@ public class GridCacheSqlQuery implements Message, GridCacheQueryMarshallable {
/**
* @param qry Query.
- * @param params Query parameters.
*/
- public GridCacheSqlQuery(String qry, Object[] params) {
+ public GridCacheSqlQuery(String qry) {
A.ensure(!F.isEmpty(qry), "qry must not be empty");
this.qry = qry;
-
- this.params = F.isEmpty(params) ? EMPTY_PARAMS : params;
- paramsSize = this.params.length;
- }
-
- /**
- * @param paramIdxs Parameter indexes.
- */
- public void parameterIndexes(int[] paramIdxs) {
- this.paramIdxs = paramIdxs;
}
/**
@@ -126,12 +115,44 @@ public class GridCacheSqlQuery implements Message, GridCacheQueryMarshallable {
}
/**
+ * @param qry Query.
+ * @return {@code this}.
+ */
+ public GridCacheSqlQuery query(String qry) {
+ this.qry = qry;
+
+ return this;
+ }
+
+ /**
* @return Parameters.
*/
public Object[] parameters() {
return params;
}
+ /**
+ * @return Parameter indexes.
+ */
+ public int[] parameterIndexes() {
+ return paramIdxs;
+ }
+
+ /**
+ * @param params Parameters.
+ * @param paramIdxs Parameter indexes.
+ * @return {@code this} For chaining.
+ */
+ public GridCacheSqlQuery parameters(Object[] params, int[] paramIdxs) {
+ this.params = F.isEmpty(params) ? EMPTY_PARAMS : params;
+
+ paramsSize = this.params.length;
+
+ this.paramIdxs = paramIdxs;
+
+ return this;
+ }
+
/** {@inheritDoc} */
@Override public void marshall(Marshaller m) {
if (paramsBytes != null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 8088f80..15e7fc6 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -214,6 +214,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
";ROW_FACTORY=\"" + GridH2RowFactory.class.getName() + "\"" +
";DEFAULT_TABLE_ENGINE=" + GridH2DefaultTableEngine.class.getName();
+ // Uncomment this setting to get debug output from H2 to sysout.
+// ";TRACE_LEVEL_SYSTEM_OUT=3";
+
/** */
private static final int PREPARED_STMT_CACHE_SIZE = 256;
@@ -1008,9 +1011,6 @@ public class IgniteH2Indexing implements GridQueryIndexing {
int timeoutMillis, @Nullable GridQueryCancel cancel)
throws IgniteCheckedException {
- if (timeoutMillis > 0)
- ((Session)((JdbcConnection)conn).getSession()).setQueryTimeout(timeoutMillis);
-
if (cancel != null) {
cancel.set(new Runnable() {
@Override public void run() {
@@ -1024,6 +1024,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
});
}
+ if (timeoutMillis > 0)
+ session(conn).setQueryTimeout(timeoutMillis);
+
try {
return stmt.executeQuery();
}
@@ -1036,7 +1039,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
finally {
if (timeoutMillis > 0)
- ((Session)((JdbcConnection)conn).getSession()).setQueryTimeout(0);
+ session(conn).setQueryTimeout(0);
}
}
@@ -1135,7 +1138,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @param distributedJoins If distributed joins are enabled.
* @param enforceJoinOrder Enforce join order of tables.
*/
- public void setupConnection(Connection conn, boolean distributedJoins, boolean enforceJoinOrder) {
+ public static void setupConnection(Connection conn, boolean distributedJoins, boolean enforceJoinOrder) {
Session s = session(conn);
s.setForceJoinOrder(enforceJoinOrder);
@@ -1278,7 +1281,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
final boolean distributedJoins = qry.isDistributedJoins() && cctx.isPartitioned();
final boolean grpByCollocated = qry.isCollocated();
- GridCacheTwoStepQuery twoStepQry;
+ GridCacheTwoStepQuery twoStepQry = null;
List<GridQueryFieldMetadata> meta;
final TwoStepCachedQueryKey cachedQryKey = new TwoStepCachedQueryKey(space, sqlQry, grpByCollocated,
@@ -1292,65 +1295,72 @@ public class IgniteH2Indexing implements GridQueryIndexing {
else {
final UUID locNodeId = ctx.localNodeId();
- setupConnection(c, distributedJoins, enforceJoinOrder);
+ // Here we will just parse the statement, no need to optimize it at all.
+ setupConnection(c, /*distributedJoins*/false, /*enforceJoinOrder*/true);
GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, 0, PREPARE)
.distributedJoins(distributedJoins));
- PreparedStatement stmt;
+ PreparedStatement stmt = null;
boolean cachesCreated = false;
try {
- while (true) {
- try {
- // Do not cache this statement because the whole two step query object will be cached later on.
- stmt = prepareStatement(c, sqlQry, false);
+ try {
+ while (true) {
+ try {
+ // Do not cache this statement because the whole two step query object will be cached later on.
+ stmt = prepareStatement(c, sqlQry, false);
- break;
- }
- catch (SQLException e) {
- if (!cachesCreated && e.getErrorCode() == ErrorCode.SCHEMA_NOT_FOUND_1) {
- try {
- ctx.cache().createMissingCaches();
- }
- catch (IgniteCheckedException ignored) {
- throw new CacheException("Failed to create missing caches.", e);
- }
+ break;
+ }
+ catch (SQLException e) {
+ if (!cachesCreated && e.getErrorCode() == ErrorCode.SCHEMA_NOT_FOUND_1) {
+ try {
+ ctx.cache().createMissingCaches();
+ }
+ catch (IgniteCheckedException ignored) {
+ throw new CacheException("Failed to create missing caches.", e);
+ }
- cachesCreated = true;
+ cachesCreated = true;
+ }
+ else
+ throw new IgniteSQLException("Failed to parse query: " + sqlQry,
+ IgniteQueryErrorCode.PARSING, e);
}
- else
- throw new IgniteSQLException("Failed to parse query: " + sqlQry,
- IgniteQueryErrorCode.PARSING, e);
}
- }
- }
- finally {
- GridH2QueryContext.clearThreadLocal();
- }
- Prepared prepared = GridSqlQueryParser.prepared((JdbcPreparedStatement) stmt);
- if (qry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery) qry).isQuery() != prepared.isQuery())
- throw new IgniteSQLException("Given statement type does not match that declared by JDBC driver",
- IgniteQueryErrorCode.STMT_TYPE_MISMATCH);
+ Prepared prepared = GridSqlQueryParser.prepared((JdbcPreparedStatement) stmt);
- if (!prepared.isQuery()) {
- try {
- return dmlProc.updateSqlFieldsTwoStep(cctx.namexx(), stmt, qry, cancel);
+ if (qry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery) qry).isQuery() != prepared.isQuery())
+ throw new IgniteSQLException("Given statement type does not match that declared by JDBC driver",
+ IgniteQueryErrorCode.STMT_TYPE_MISMATCH);
+
+ if (prepared.isQuery()) {
+ bindParameters(stmt, F.asList(qry.getArgs()));
+
+ twoStepQry = GridSqlQuerySplitter.split((JdbcPreparedStatement)stmt, qry.getArgs(), grpByCollocated,
+ distributedJoins, enforceJoinOrder, this);
+
+ assert twoStepQry != null;
+ }
}
- catch (IgniteCheckedException e) {
- throw new IgniteSQLException("Failed to execute DML statement [qry=" + sqlQry + ", params=" +
- Arrays.deepToString(qry.getArgs()) + "]", e);
+ finally {
+ GridH2QueryContext.clearThreadLocal();
}
- }
-
- try {
- bindParameters(stmt, F.asList(qry.getArgs()));
- twoStepQry = GridSqlQuerySplitter.split((JdbcPreparedStatement)stmt, qry.getArgs(), grpByCollocated,
- distributedJoins);
+ // It is a DML statement if we did not create a twoStepQuery.
+ if (twoStepQry == null) {
+ try {
+ return dmlProc.updateSqlFieldsTwoStep(cctx.namexx(), stmt, qry, cancel);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteSQLException("Failed to execute DML statement [qry=" + sqlQry + ", params=" +
+ Arrays.deepToString(qry.getArgs()) + "]", e);
+ }
+ }
List<Integer> caches;
List<Integer> extraCaches = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
index fdcd164..fc82161 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
@@ -76,7 +76,7 @@ public final class UpdatePlanBuilder {
@Nullable Integer errKeysPos) throws IgniteCheckedException {
assert !prepared.isQuery();
- GridSqlStatement stmt = new GridSqlQueryParser().parse(prepared);
+ GridSqlStatement stmt = new GridSqlQueryParser(false).parse(prepared);
if (stmt instanceof GridSqlMerge || stmt instanceof GridSqlInsert)
return planForInsert(stmt);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java
index 0f76316..ce11fd5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2CollocationModel.java
@@ -23,6 +23,7 @@ import java.util.List;
import javax.cache.CacheException;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.SB;
import org.h2.command.dml.Query;
import org.h2.command.dml.Select;
import org.h2.command.dml.SelectUnion;
@@ -38,6 +39,8 @@ import org.h2.table.Table;
import org.h2.table.TableFilter;
import org.h2.table.TableView;
+import static org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.KEY_COL;
+
/**
* Collocation model for a query.
*/
@@ -98,6 +101,63 @@ public final class GridH2CollocationModel {
}
/**
+ * @return Table filter for this collocation model.
+ */
+ private TableFilter filter() {
+ return upper == null ? null : upper.childFilters[filter];
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ calculate();
+
+ SB b = new SB();
+
+ for (int lvl = 0; lvl < 20; lvl++) {
+ if (!toString(b, lvl))
+ break;
+
+ b.a('\n');
+ }
+
+ return b.toString();
+ }
+
+ /**
+ * @param b String builder.
+ * @param lvl Depth level.
+ */
+ private boolean toString(SB b, int lvl) {
+ boolean res = false;
+
+ if (lvl == 0) {
+ TableFilter f = filter();
+ String tblAlias = f == null ? "^" : f.getTableAlias();
+
+ b.a("[tbl=").a(tblAlias).a(", type=").a(type).a(", mul=").a(multiplier).a("]");
+
+ res = true;
+ }
+ else if (childFilters != null) {
+ assert lvl > 0;
+
+ lvl--;
+
+ for (int i = 0; i < childFilters.length; i++) {
+ if (lvl == 0)
+ b.a(" | ");
+
+ res |= child(i, true).toString(b, lvl);
+ }
+
+ if (lvl == 0)
+ b.a(" | ");
+ }
+
+ return res;
+ }
+
+ /**
* @param upper Upper.
* @param filter Filter.
* @param unions Unions.
@@ -240,7 +300,7 @@ public final class GridH2CollocationModel {
assert childFilters == null;
// We are at table instance.
- GridH2Table tbl = (GridH2Table)upper.childFilters[filter].getTable();
+ GridH2Table tbl = (GridH2Table)filter().getTable();
// Only partitioned tables will do distributed joins.
if (!tbl.isPartitioned()) {
@@ -261,7 +321,7 @@ public final class GridH2CollocationModel {
// It is enough to make sure that our previous join by affinity key is collocated, then we are
// collocated. If we at least have affinity key condition, then we do unicast which is cheaper.
switch (upper.joinedWithCollocated(filter)) {
- case JOINED_WITH_COLLOCATED:
+ case COLLOCATED_JOIN:
type = Type.PARTITIONED_COLLOCATED;
multiplier = MULTIPLIER_COLLOCATED;
@@ -349,15 +409,14 @@ public final class GridH2CollocationModel {
for (int i = 0; i < idxConditions.size(); i++) {
IndexCondition c = idxConditions.get(i);
-
+ int colId = c.getColumn().getColumnId();
int cmpType = c.getCompareType();
if ((cmpType == Comparison.EQUAL || cmpType == Comparison.EQUAL_NULL_SAFE) &&
- c.getColumn().getColumnId() == affColId && c.isEvaluatable()) {
+ (colId == affColId || colId == KEY_COL) && c.isEvaluatable()) {
affKeyCondFound = true;
Expression exp = c.getExpression();
-
exp = exp.getNonAliasExpression();
if (exp instanceof ExpressionColumn) {
@@ -369,11 +428,14 @@ public final class GridH2CollocationModel {
if (prevJoin != null) {
GridH2CollocationModel cm = child(indexOf(prevJoin), true);
- if (cm != null) {
+ // If the previous joined model is a subquery (view), we can not be sure that
+ // the found affinity column is the needed one, since we can select multiple
+ // different affinity columns from different tables.
+ if (cm != null && !cm.view) {
Type t = cm.type(true);
if (t.isPartitioned() && t.isCollocated() && isAffinityColumn(prevJoin, expCol, validate))
- return Affinity.JOINED_WITH_COLLOCATED;
+ return Affinity.COLLOCATED_JOIN;
}
}
}
@@ -778,6 +840,6 @@ public final class GridH2CollocationModel {
HAS_AFFINITY_CONDITION,
/** */
- JOINED_WITH_COLLOCATED
+ COLLOCATED_JOIN
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index bab219c..bdfddd5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -245,14 +245,15 @@ public abstract class GridH2IndexBase extends BaseIndex {
* @param filter Current filter.
* @return Multiplier.
*/
- public int getDistributedMultiplier(Session ses, TableFilter[] filters, int filter) {
+ public final int getDistributedMultiplier(Session ses, TableFilter[] filters, int filter) {
GridH2QueryContext qctx = GridH2QueryContext.get();
- // We do complex optimizations with respect to distributed joins only on prepare stage
- // because on run stage reordering of joined tables by Optimizer is explicitly disabled
- // and thus multiplier will be always the same, so it will not affect choice of index.
+ // We do optimizations with respect to distributed joins only on PREPARE stage only.
+ // Notice that we check for isJoinBatchEnabled, because we can do multiple different
+ // optimization passes on PREPARE stage.
// Query expressions can not be distributed as well.
- if (qctx == null || qctx.type() != PREPARE || !qctx.distributedJoins() || ses.isPreparingQueryExpression())
+ if (qctx == null || qctx.type() != PREPARE || !qctx.distributedJoins() ||
+ !ses.isJoinBatchEnabled() || ses.isPreparingQueryExpression())
return GridH2CollocationModel.MULTIPLIER_COLLOCATED;
// We have to clear this cache because normally sub-query plan cost does not depend on anything
@@ -368,17 +369,17 @@ public abstract class GridH2IndexBase extends BaseIndex {
IndexColumn affCol = getTable().getAffinityKeyColumn();
- int affColId;
- boolean ucast;
+ int affColId = -1;
+ boolean ucast = false;
if (affCol != null) {
affColId = affCol.column.getColumnId();
int[] masks = filter.getMasks();
- ucast = masks != null && masks[affColId] == IndexCondition.EQUALITY;
- }
- else {
- affColId = -1;
- ucast = false;
+
+ if (masks != null) {
+ ucast = (masks[affColId] & IndexCondition.EQUALITY) != 0 ||
+ (masks[KEY_COL] & IndexCondition.EQUALITY) != 0;
+ }
}
GridCacheContext<?,?> cctx = getTable().rowDescriptor().context();
@@ -713,6 +714,21 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
/**
+ * @param arr Array.
+ * @param off Offset.
+ * @param cmp Comparator.
+ */
+ public static <Z> void bubbleUp(Z[] arr, int off, Comparator<Z> cmp) {
+ // TODO Optimize: use binary search if the range in array is big.
+ for (int i = off, last = arr.length - 1; i < last; i++) {
+ if (cmp.compare(arr[i], arr[i + 1]) <= 0)
+ break;
+
+ U.swap(arr, i, i + 1);
+ }
+ }
+
+ /**
* @param msg Message.
* @return Row.
*/
@@ -877,12 +893,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
// Bubble up current min stream with respect to fetched row to achieve correct sort order of streams.
- for (int i = off, last = streams.length - 1; i < last; i++) {
- if (compareRows(streams[i].get(rangeId), streams[i + 1].get(rangeId)) <= 0)
- break;
-
- U.swap(streams, i, i + 1);
- }
+ bubbleUp(streams, off, this);
return true;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
index 9752c5f..35f0983 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
@@ -95,7 +95,7 @@ public final class DmlAstUtils {
args[i] = arr;
- GridSqlColumn newCol = new GridSqlColumn(null, from, colName, "TABLE." + colName);
+ GridSqlColumn newCol = new GridSqlColumn(null, from, null,"TABLE", colName);
newCol.resultType(cols[i].resultType());
@@ -146,10 +146,10 @@ public final class DmlAstUtils {
Column h2ValCol = gridTbl.getColumn(GridH2AbstractKeyValueRow.VAL_COL);
- GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName(), h2KeyCol.getSQL());
+ GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName());
keyCol.resultType(GridSqlType.fromColumn(h2KeyCol));
- GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName(), h2ValCol.getSQL());
+ GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName());
valCol.resultType(GridSqlType.fromColumn(h2ValCol));
mapQry.addColumn(keyCol, true);
@@ -234,7 +234,7 @@ public final class DmlAstUtils {
// Does this WHERE limit only by _key?
if (isKeyEqualityCondition(whereOp))
- return new IgnitePair<>(whereOp.child(1), null);
+ return new IgnitePair<>((GridSqlElement)whereOp.child(1), null);
// Or maybe it limits both by _key and _val?
if (whereOp.operationType() != GridSqlOperationType.AND)
@@ -255,13 +255,13 @@ public final class DmlAstUtils {
if (!isValueEqualityCondition(rightOp))
return null;
- return new IgnitePair<>(leftOp.child(1), rightOp.child(1));
+ return new IgnitePair<>((GridSqlElement)leftOp.child(1), (GridSqlElement)rightOp.child(1));
}
else if (isKeyEqualityCondition(rightOp)) { // _val = ? and _key = ?
if (!isValueEqualityCondition(leftOp))
return null;
- return new IgnitePair<>(rightOp.child(1), leftOp.child(1));
+ return new IgnitePair<>((GridSqlElement)rightOp.child(1), (GridSqlElement)leftOp.child(1));
}
else // Neither
return null;
@@ -329,10 +329,10 @@ public final class DmlAstUtils {
Column h2ValCol = gridTbl.getColumn(GridH2AbstractKeyValueRow.VAL_COL);
- GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName(), h2KeyCol.getSQL());
+ GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName());
keyCol.resultType(GridSqlType.fromColumn(h2KeyCol));
- GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName(), h2ValCol.getSQL());
+ GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName());
valCol.resultType(GridSqlType.fromColumn(h2ValCol));
mapQry.addColumn(keyCol, true);
@@ -424,7 +424,7 @@ public final class DmlAstUtils {
sel.from(from);
- GridSqlColumn col = new GridSqlColumn(null, from, "_IGNITE_ERR_KEYS", "TABLE._IGNITE_ERR_KEYS");
+ GridSqlColumn col = new GridSqlColumn(null, from, null, "TABLE", "_IGNITE_ERR_KEYS");
sel.addColumn(col, true);
@@ -459,8 +459,8 @@ public final class DmlAstUtils {
findParams(union.left(), params, target, paramIdxs);
findParams(union.right(), params, target, paramIdxs);
- findParams(qry.limit(), params, target, paramIdxs);
- findParams(qry.offset(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.limit(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.offset(), params, target, paramIdxs);
return target;
}
@@ -477,16 +477,16 @@ public final class DmlAstUtils {
if (params.length == 0)
return target;
- for (GridSqlElement el : qry.columns(false))
- findParams(el, params, target, paramIdxs);
+ for (GridSqlAst el : qry.columns(false))
+ findParams((GridSqlElement)el, params, target, paramIdxs);
- findParams(qry.from(), params, target, paramIdxs);
- findParams(qry.where(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.from(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.where(), params, target, paramIdxs);
// Don't search in GROUP BY and HAVING since they expected to be in select list.
- findParams(qry.limit(), params, target, paramIdxs);
- findParams(qry.offset(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.limit(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.offset(), params, target, paramIdxs);
return target;
}
@@ -524,10 +524,10 @@ public final class DmlAstUtils {
paramIdxs.add(idx);
}
else if (el instanceof GridSqlSubquery)
- findParams(((GridSqlSubquery)el).select(), params, target, paramIdxs);
+ findParams(((GridSqlSubquery)el).subquery(), params, target, paramIdxs);
else
- for (GridSqlElement child : el)
- findParams(child, params, target, paramIdxs);
+ for (int i = 0; i < el.size(); i++)
+ findParams((GridSqlElement)el.child(i), params, target, paramIdxs);
}
/**
@@ -546,17 +546,17 @@ public final class DmlAstUtils {
if (from instanceof GridSqlJoin) {
// Left and right.
- if (findTablesInFrom(from.child(0), c))
+ if (findTablesInFrom((GridSqlElement)from.child(0), c))
return true;
- if (findTablesInFrom(from.child(1), c))
+ if (findTablesInFrom((GridSqlElement)from.child(1), c))
return true;
// We don't process ON condition because it is not a joining part of from here.
return false;
}
else if (from instanceof GridSqlAlias)
- return findTablesInFrom(from.child(), c);
+ return findTablesInFrom((GridSqlElement)from.child(), c);
else if (from instanceof GridSqlFunction)
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAlias.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAlias.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAlias.java
index 3fb6f3c..eb84c90 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAlias.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAlias.java
@@ -34,7 +34,7 @@ public class GridSqlAlias extends GridSqlElement {
* @param alias Alias.
* @param expr Expr.
*/
- public GridSqlAlias(String alias, GridSqlElement expr) {
+ public GridSqlAlias(String alias, GridSqlAst expr) {
this(alias, expr, false);
}
@@ -43,8 +43,8 @@ public class GridSqlAlias extends GridSqlElement {
* @param expr Expr.
* @param useAs Use 'AS' keyword.
*/
- public GridSqlAlias(String alias, GridSqlElement expr, boolean useAs) {
- super(new ArrayList<GridSqlElement>(1));
+ public GridSqlAlias(String alias, GridSqlAst expr, boolean useAs) {
+ super(new ArrayList<GridSqlAst>(1));
addChild(expr);
@@ -56,17 +56,18 @@ public class GridSqlAlias extends GridSqlElement {
* @param el Element.
* @return Unwrapped from alias element.
*/
- public static GridSqlElement unwrap(GridSqlElement el) {
+ @SuppressWarnings("unchecked")
+ public static <X extends GridSqlAst> X unwrap(GridSqlAst el) {
el = el instanceof GridSqlAlias ? el.child() : el;
assert el != null;
- return el;
+ return (X)el;
}
/** {@inheritDoc} */
@Override public String getSQL() {
- return child().getSQL() + (useAs ? " AS " : " ") + Parser.quoteIdentifier(alias);
+ return child(0).getSQL() + (useAs ? " AS " : " ") + Parser.quoteIdentifier(alias);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlArray.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlArray.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlArray.java
index caa2000..2f7e2f0 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlArray.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlArray.java
@@ -30,13 +30,13 @@ public class GridSqlArray extends GridSqlElement {
* @param size Array size.
*/
public GridSqlArray(int size) {
- super(size == 0 ? Collections.<GridSqlElement>emptyList() : new ArrayList<GridSqlElement>(size));
+ super(size == 0 ? Collections.<GridSqlAst>emptyList() : new ArrayList<GridSqlAst>(size));
}
/**
* @param children Initial child list.
*/
- public GridSqlArray(List<GridSqlElement> children) {
+ public GridSqlArray(List<GridSqlAst> children) {
super(children);
}
@@ -47,9 +47,9 @@ public class GridSqlArray extends GridSqlElement {
StatementBuilder buff = new StatementBuilder("(");
- for (GridSqlElement e : children) {
+ for (int i = 0; i < size(); i++) {
buff.appendExceptFirst(", ");
- buff.append(e.getSQL());
+ buff.append(child(i).getSQL());
}
if (size() == 1)
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAst.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAst.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAst.java
new file mode 100644
index 0000000..b087e74
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlAst.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sql;
+
+/**
+ * AST for SQL.
+ */
+public interface GridSqlAst {
+ /**
+ * @return Generate sql from this AST.
+ */
+ public String getSQL();
+
+ /**
+ * @return Number of child nodes.
+ */
+ public int size();
+
+ /**
+ * Get child by index.
+ *
+ * @param childIdx Index of the requested child.
+ * @return Child element.
+ */
+ public <E extends GridSqlAst> E child(int childIdx);
+
+ /**
+ * Get the first child.
+ *
+ * @return Child element.
+ */
+ public <E extends GridSqlAst> E child();
+
+ /**
+ * Set child.
+ *
+ * @param childIdx Index of the requested child.
+ * @param child Child element.
+ */
+ public <E extends GridSqlAst> void child(int childIdx, E child);
+
+ /**
+ * @return Optional expression result type (if this is an expression and result type is known).
+ */
+ public GridSqlType resultType();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
index 6ef4446..6f4141e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlColumn.java
@@ -18,20 +18,25 @@
package org.apache.ignite.internal.processors.query.h2.sql;
import java.util.Collections;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.command.Parser;
import org.h2.table.Column;
/**
* Column.
*/
-public class GridSqlColumn extends GridSqlElement implements GridSqlValue {
+public class GridSqlColumn extends GridSqlElement {
/** */
- private final GridSqlElement expressionInFrom;
+ private GridSqlAst from;
/** */
- private final String colName;
+ private String schema;
- /** SQL from original query. May be qualified or unqualified column name. */
- private final String sqlText;
+ /** */
+ private String tblAlias;
+
+ /** */
+ private String colName;
/** */
private Column col;
@@ -39,18 +44,30 @@ public class GridSqlColumn extends GridSqlElement implements GridSqlValue {
/**
* @param col Column.
* @param from From.
- * @param name Name.
- * @param sqlText Text.
+ * @param colName Column name.
*/
- public GridSqlColumn(Column col, GridSqlElement from, String name, String sqlText) {
- super(Collections.<GridSqlElement>emptyList());
+ public GridSqlColumn(Column col, GridSqlAst from, String colName) {
+ this(col, from, null, null, colName);
+ }
- assert sqlText != null;
+ /**
+ * @param col Column.
+ * @param from From.
+ * @param schema Schema name.
+ * @param tblAlias Table alias.
+ * @param colName Column name.
+ */
+ public GridSqlColumn(Column col, GridSqlAst from, String schema, String tblAlias, String colName) {
+ super(Collections.<GridSqlAst>emptyList());
+
+ assert !F.isEmpty(colName): colName;
- expressionInFrom = from;
- colName = name;
- this.sqlText = sqlText;
this.col = col;
+ this.from = from;
+
+ this.colName = colName;
+ this.schema = schema;
+ this.tblAlias = tblAlias;
}
/**
@@ -60,16 +77,52 @@ public class GridSqlColumn extends GridSqlElement implements GridSqlValue {
return colName;
}
+ /**
+ * @return Schema name.
+ */
+ public String schema() {
+ return schema;
+ }
+
+ /**
+ * @return Table alias.
+ */
+ public String tableAlias() {
+ return tblAlias;
+ }
+
+ /**
+ * @param tblAlias Table alias.
+ */
+ public void tableAlias(String tblAlias) {
+ this.tblAlias = tblAlias;
+ }
+
/** {@inheritDoc} */
@Override public String getSQL() {
- return sqlText;
+ String sql = Parser.quoteIdentifier(colName);
+
+ if (tblAlias != null)
+ sql = Parser.quoteIdentifier(tblAlias) + "." + sql;
+
+ if (schema != null)
+ sql = Parser.quoteIdentifier(schema) + "." + sql;
+
+ return sql;
}
/**
* @return Expression in from.
*/
- public GridSqlElement expressionInFrom() {
- return expressionInFrom;
+ public GridSqlAst expressionInFrom() {
+ return from;
+ }
+
+ /**
+ * @param from Expression in from.
+ */
+ public void expressionInFrom(GridSqlAlias from) {
+ this.from = from;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
index bb7d338..0d879a8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlConst.java
@@ -26,13 +26,13 @@ import org.h2.value.ValueNull;
/**
* Constant value.
*/
-public class GridSqlConst extends GridSqlElement implements GridSqlValue {
+public class GridSqlConst extends GridSqlElement {
/** */
public static final GridSqlElement NULL = new GridSqlConst(ValueNull.INSTANCE)
.resultType(GridSqlType.fromExpression(ValueExpression.getNull()));
/** */
- public static final GridSqlElement TRUE = new GridSqlConst(ValueBoolean.get(true))
+ public static final GridSqlConst TRUE = (GridSqlConst)new GridSqlConst(ValueBoolean.get(true))
.resultType(GridSqlType.BOOLEAN);
/** */
@@ -42,7 +42,7 @@ public class GridSqlConst extends GridSqlElement implements GridSqlValue {
* @param val Value.
*/
public GridSqlConst(Value val) {
- super(Collections.<GridSqlElement>emptyList());
+ super(Collections.<GridSqlAst>emptyList());
this.val = val;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlElement.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlElement.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlElement.java
index d95c14a..00a8f22 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlElement.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlElement.java
@@ -17,15 +17,14 @@
package org.apache.ignite.internal.processors.query.h2.sql;
-import java.util.Iterator;
import java.util.List;
/**
- * Abstract SQL element.
+ * Base class for all SQL AST nodes.
*/
-public abstract class GridSqlElement implements Iterable<GridSqlElement> {
+public abstract class GridSqlElement implements GridSqlAst {
/** */
- protected final List<GridSqlElement> children;
+ private final List<GridSqlAst> children;
/** */
private GridSqlType resultType;
@@ -33,7 +32,7 @@ public abstract class GridSqlElement implements Iterable<GridSqlElement> {
/**
* @param children Initial child list.
*/
- protected GridSqlElement(List<GridSqlElement> children) {
+ protected GridSqlElement(List<GridSqlAst> children) {
assert children != null;
this.children = children;
@@ -57,17 +56,10 @@ public abstract class GridSqlElement implements Iterable<GridSqlElement> {
}
/**
- * Get the SQL expression.
- *
- * @return the SQL expression.
- */
- public abstract String getSQL();
-
- /**
* @param expr Expr.
* @return {@code this}.
*/
- public GridSqlElement addChild(GridSqlElement expr) {
+ public GridSqlElement addChild(GridSqlAst expr) {
if (expr == null)
throw new NullPointerException();
@@ -76,27 +68,19 @@ public abstract class GridSqlElement implements Iterable<GridSqlElement> {
return this;
}
- /**
- * @return First child.
- */
- public <E extends GridSqlElement> E child() {
+ /** {@inheritDoc} */
+ @Override public <E extends GridSqlAst> E child() {
return child(0);
}
- /**
- * @param idx Index.
- * @return Child.
- */
+ /** {@inheritDoc} */
@SuppressWarnings("unchecked")
- public <E extends GridSqlElement> E child(int idx) {
+ @Override public <E extends GridSqlAst> E child(int idx) {
return (E)children.get(idx);
}
- /**
- * @param idx Index.
- * @param child New child.
- */
- public void child(int idx, GridSqlElement child) {
+ /** {@inheritDoc} */
+ @Override public <E extends GridSqlAst> void child(int idx, E child) {
if (child == null)
throw new NullPointerException();
@@ -111,11 +95,6 @@ public abstract class GridSqlElement implements Iterable<GridSqlElement> {
}
/** {@inheritDoc} */
- @Override public Iterator<GridSqlElement> iterator() {
- return children.iterator();
- }
-
- /** {@inheritDoc} */
@Override public String toString() {
return getSQL();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
index 4e93196..f194b07 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlFunction.java
@@ -66,7 +66,7 @@ public class GridSqlFunction extends GridSqlElement {
* @param name Name.
*/
private GridSqlFunction(String schema, GridSqlFunctionType type, String name) {
- super(new ArrayList<GridSqlElement>());
+ super(new ArrayList<GridSqlAst>());
if (name == null)
throw new NullPointerException("name");
@@ -102,13 +102,13 @@ public class GridSqlFunction extends GridSqlElement {
if (type == CASE) {
buff.append(' ').append(child().getSQL());
- for (int i = 1, len = children.size() - 1; i < len; i += 2) {
+ for (int i = 1, len = size() - 1; i < len; i += 2) {
buff.append(" WHEN ").append(child(i).getSQL());
buff.append(" THEN ").append(child(i + 1).getSQL());
}
- if ((children.size() & 1) == 0)
- buff.append(" ELSE ").append(child(children.size() - 1).getSQL());
+ if ((size() & 1) == 0)
+ buff.append(" ELSE ").append(child(size() - 1).getSQL());
return buff.append(" END").toString();
}
@@ -137,9 +137,11 @@ public class GridSqlFunction extends GridSqlElement {
break;
case TABLE:
- for (GridSqlElement e : children) {
+ for (int i = 0; i < size(); i++) {
buff.appendExceptFirst(", ");
+ GridSqlElement e = child(i);
+
// id int = ?, name varchar = ('aaa', 'bbb')
buff.append(((GridSqlAlias)e).alias())
.append(' ')
@@ -151,9 +153,9 @@ public class GridSqlFunction extends GridSqlElement {
break;
default:
- for (GridSqlElement e : children) {
+ for (int i = 0; i < size(); i++) {
buff.appendExceptFirst(", ");
- buff.append(e.getSQL());
+ buff.append(child(i).getSQL());
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
index f1ad2e5..afe4a53 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlJoin.java
@@ -27,6 +27,15 @@ import org.jetbrains.annotations.Nullable;
*/
public class GridSqlJoin extends GridSqlElement {
/** */
+ public static final int LEFT_TABLE_CHILD = 0;
+
+ /** */
+ public static final int RIGHT_TABLE_CHILD = 1;
+
+ /** */
+ public static final int ON_CHILD = 2;
+
+ /** */
private boolean leftOuter;
/**
@@ -36,7 +45,7 @@ public class GridSqlJoin extends GridSqlElement {
* @param on Join condition.
*/
public GridSqlJoin(GridSqlElement leftTbl, GridSqlElement rightTbl, boolean leftOuter, @Nullable GridSqlElement on) {
- super(new ArrayList<GridSqlElement>(3));
+ super(new ArrayList<GridSqlAst>(3));
addChild(leftTbl);
addChild(rightTbl);
@@ -50,24 +59,38 @@ public class GridSqlJoin extends GridSqlElement {
}
/**
- * @return Table 1.
+ * @return Left table.
*/
public GridSqlElement leftTable() {
- return child(0);
+ return child(LEFT_TABLE_CHILD);
+ }
+
+ /**
+ * @param tbl Right table to set.
+ */
+ public void leftTable(GridSqlElement tbl) {
+ child(LEFT_TABLE_CHILD, tbl);
}
/**
- * @return Table 2.
+ * @return Right table.
*/
public GridSqlElement rightTable() {
- return child(1);
+ return child(RIGHT_TABLE_CHILD);
+ }
+
+ /**
+ * @param tbl Right table to set.
+ */
+ public void rightTable(GridSqlElement tbl) {
+ child(RIGHT_TABLE_CHILD, tbl);
}
/**
* @return {@code JOIN ON} condition.
*/
public GridSqlElement on() {
- return child(2);
+ return child(ON_CHILD);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlKeyword.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlKeyword.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlKeyword.java
index 8eacf02..48fc8a9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlKeyword.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlKeyword.java
@@ -35,7 +35,8 @@ public final class GridSqlKeyword extends GridSqlElement {
/** */
private GridSqlKeyword(String keyword) {
- super(Collections.<GridSqlElement>emptyList());
+ super(Collections.<GridSqlAst>emptyList());
+
this.keyword = keyword;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperation.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperation.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperation.java
index 737c5b1..4fcca16 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperation.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperation.java
@@ -32,8 +32,8 @@ public class GridSqlOperation extends GridSqlElement {
* @param opType Operation type.
*/
public GridSqlOperation(GridSqlOperationType opType) {
- super(opType == IN ? new ArrayList<GridSqlElement>() :
- new ArrayList<GridSqlElement>(opType.childrenCount()));
+ super(opType == IN ? new ArrayList<GridSqlAst>() :
+ new ArrayList<GridSqlAst>(opType.childrenCount()));
this.opType = opType;
}
@@ -53,7 +53,7 @@ public class GridSqlOperation extends GridSqlElement {
* @param left Left.
* @param right Right.
*/
- public GridSqlOperation(GridSqlOperationType opType, GridSqlElement left, GridSqlElement right) {
+ public GridSqlOperation(GridSqlOperationType opType, GridSqlAst left, GridSqlAst right) {
this(opType);
addChild(left);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
index 8d31651..07a6f6b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlOperationType.java
@@ -156,7 +156,7 @@ public enum GridSqlOperationType {
@Override public String getSql(GridSqlOperation operation) {
assert operation.operationType().childrenCnt == 1;
- return '(' + text + ' ' + operation.child().getSQL() + ')';
+ return '(' + text + ' ' + operation.child(0).getSQL() + ')';
}
}
@@ -178,7 +178,7 @@ public enum GridSqlOperationType {
@Override public String getSql(GridSqlOperation operation) {
assert operation.operationType().childrenCnt == 1;
- return '(' + operation.child().getSQL() + ' ' + text + ')';
+ return '(' + operation.child(0).getSQL() + ' ' + text + ')';
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlParameter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlParameter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlParameter.java
index 0969ad2..f19e9f5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlParameter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlParameter.java
@@ -22,7 +22,7 @@ import java.util.Collections;
/**
* Query parameter.
*/
-public class GridSqlParameter extends GridSqlElement implements GridSqlValue {
+public class GridSqlParameter extends GridSqlElement {
/** Index. */
private int idx;
@@ -30,7 +30,7 @@ public class GridSqlParameter extends GridSqlElement implements GridSqlValue {
* @param idx Index.
*/
public GridSqlParameter(int idx) {
- super(Collections.<GridSqlElement>emptyList());
+ super(Collections.<GridSqlAst>emptyList());
this.idx = idx;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlPlaceholder.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlPlaceholder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlPlaceholder.java
index 0bb69a8..3798396 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlPlaceholder.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlPlaceholder.java
@@ -33,7 +33,7 @@ public class GridSqlPlaceholder extends GridSqlElement {
* @param sql SQL.
*/
public GridSqlPlaceholder(String sql) {
- super(Collections.<GridSqlElement>emptyList());
+ super(Collections.<GridSqlAst>emptyList());
this.sql = sql;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
index 331aa65..7d4b7f0 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuery.java
@@ -23,29 +23,35 @@ import org.h2.util.StatementBuilder;
import org.h2.util.StringUtils;
/**
- * Select query.
+ * SQL Query AST.
*/
-public abstract class GridSqlQuery extends GridSqlStatement {
+public abstract class GridSqlQuery extends GridSqlStatement implements GridSqlAst {
/** */
- protected boolean distinct;
+ public static final int OFFSET_CHILD = 0;
+
+ /** */
+ public static final int LIMIT_CHILD = 1;
/** */
protected List<GridSqlSortColumn> sort = new ArrayList<>();
/** */
- protected GridSqlElement offset;
+ private GridSqlAst offset;
+
+ /** */
+ private boolean distinct;
/**
* @return Offset.
*/
- public GridSqlElement offset() {
+ public GridSqlAst offset() {
return offset;
}
/**
* @param offset Offset.
*/
- public void offset(GridSqlElement offset) {
+ public void offset(GridSqlAst offset) {
this.offset = offset;
}
@@ -93,7 +99,63 @@ public abstract class GridSqlQuery extends GridSqlStatement {
* @param col Column index.
* @return Expression for column index.
*/
- protected abstract GridSqlElement column(int col);
+ protected abstract GridSqlAst column(int col);
+
+ /** {@inheritDoc} */
+ @Override public GridSqlType resultType() {
+ return GridSqlType.RESULT_SET;
+ }
+
+ /** {@inheritDoc} */
+ @Override public <E extends GridSqlAst> E child() {
+ return child(0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <E extends GridSqlAst> E child(int childIdx) {
+ switch (childIdx) {
+ case OFFSET_CHILD:
+ return maskNull(offset, GridSqlPlaceholder.EMPTY);
+
+ case LIMIT_CHILD:
+ return maskNull(limit, GridSqlPlaceholder.EMPTY);
+
+ default:
+ throw new IllegalStateException("Child index: " + childIdx);
+ }
+ }
+
+ /**
+ * @param x Element.
+ * @return Empty placeholder if the element is {@code null}.
+ */
+ @SuppressWarnings("unchecked")
+ protected static <E extends GridSqlAst> E maskNull(GridSqlAst x, GridSqlAst dflt) {
+ return (E)(x == null ? dflt : x);
+ }
+
+ /** {@inheritDoc} */
+ @Override public <E extends GridSqlAst> void child(int childIdx, E child) {
+ switch (childIdx) {
+ case OFFSET_CHILD:
+ offset = child;
+
+ break;
+
+ case LIMIT_CHILD:
+ limit = child;
+
+ break;
+
+ default:
+ throw new IllegalStateException("Child index: " + childIdx);
+ }
+ }
+
+ /**
+ * @return If this is a simple query with no conditions, expressions, sorting, etc...
+ */
+ public abstract boolean simpleQuery();
/**
* @param buff Statement builder.
@@ -116,13 +178,13 @@ public abstract class GridSqlQuery extends GridSqlStatement {
if (idx < visibleCols)
buff.append(idx + 1);
else {
- GridSqlElement expr = column(idx);
+ GridSqlAst expr = column(idx);
if (expr == null) // For plain select should never be null, for union H2 itself can't parse query.
throw new IllegalStateException("Failed to build query: " + buff.toString());
if (expr instanceof GridSqlAlias)
- expr = expr.child();
+ expr = expr.child(0);
buff.append('=').append(StringUtils.unEnclose(expr.getSQL()));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index d9c546c..f5ad397 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.query.h2.sql;
import java.lang.reflect.Field;
+import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -27,8 +28,6 @@ import java.util.List;
import java.util.Map;
import javax.cache.CacheException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
-import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.h2.command.Command;
import org.h2.command.CommandContainer;
import org.h2.command.Prepared;
@@ -60,7 +59,6 @@ import org.h2.expression.Parameter;
import org.h2.expression.Subquery;
import org.h2.expression.TableFunction;
import org.h2.expression.ValueExpression;
-import org.h2.index.Index;
import org.h2.index.ViewIndex;
import org.h2.jdbc.JdbcPreparedStatement;
import org.h2.result.SortOrder;
@@ -225,6 +223,9 @@ public class GridSqlQueryParser {
private static final Getter<JavaFunction, FunctionAlias> FUNC_ALIAS = getter(JavaFunction.class, "functionAlias");
/** */
+ private static final Getter<ExpressionColumn, String> SCHEMA_NAME = getter(ExpressionColumn.class, "schemaName");
+
+ /** */
private static final Getter<JdbcPreparedStatement, Command> COMMAND = getter(JdbcPreparedStatement.class, "command");
/** */
@@ -298,12 +299,30 @@ public class GridSqlQueryParser {
/** */
private final IdentityHashMap<Object, Object> h2ObjToGridObj = new IdentityHashMap<>();
+ /** */
+ private final Map<String, Integer> optimizedTableFilterOrder;
+
+ /**
+ * We have a counter instead of a simple flag, because
+ * a flag can be reset earlier than needed in case of
+ * deep subquery expression nesting.
+ */
+ private int parsingSubQryExpression;
+
+ /**
+ * @param useOptimizedSubqry If we have to find correct order for table filters in FROM clause.
+ * Relies on uniqueness of table filter aliases.
+ */
+ public GridSqlQueryParser(boolean useOptimizedSubqry) {
+ optimizedTableFilterOrder = useOptimizedSubqry ? new HashMap<String, Integer>() : null;
+ }
+
/**
* @param stmt Prepared statement.
* @return Parsed select.
*/
- public static Prepared prepared(JdbcPreparedStatement stmt) {
- Command cmd = COMMAND.get(stmt);
+ public static Prepared prepared(PreparedStatement stmt) {
+ Command cmd = COMMAND.get((JdbcPreparedStatement)stmt);
assert cmd instanceof CommandContainer;
@@ -311,14 +330,25 @@ public class GridSqlQueryParser {
}
/**
+ * @param qry Query expression to parse.
+ * @return Subquery AST.
+ */
+ private GridSqlSubquery parseQueryExpression(Query qry) {
+ parsingSubQryExpression++;
+ GridSqlQuery subQry = parseQuery(qry);
+ parsingSubQryExpression--;
+
+ return new GridSqlSubquery(subQry);
+ }
+
+ /**
* @param filter Filter.
*/
private GridSqlElement parseTable(TableFilter filter) {
GridSqlElement res = (GridSqlElement)h2ObjToGridObj.get(filter);
if (res == null) {
- res = parseTable(filter.getTable(), filter.getIndex(),
- filter.getSelect() != null ? filter.getSelect().getSQL() : null);
+ res = parseTable(filter.getTable());
String alias = ALIAS.get(filter);
@@ -335,7 +365,7 @@ public class GridSqlQueryParser {
/**
* @param tbl Table.
*/
- private GridSqlElement parseTable(Table tbl, @Nullable Index idx, String sql) {
+ private GridSqlElement parseTable(Table tbl) {
GridSqlElement res = (GridSqlElement)h2ObjToGridObj.get(tbl);
if (res == null) {
@@ -344,9 +374,7 @@ public class GridSqlQueryParser {
else if (tbl instanceof TableView) {
Query qry = VIEW_QUERY.get((TableView) tbl);
- Query idxQry = idx instanceof ViewIndex ? ((ViewIndex)idx).getQuery() : null;
-
- res = new GridSqlSubquery(parse(qry, idxQry));
+ res = new GridSqlSubquery(parseQuery(qry));
}
else if (tbl instanceof FunctionTable)
res = parseExpression(FUNC_EXPR.get((FunctionTable)tbl), false);
@@ -357,8 +385,7 @@ public class GridSqlQueryParser {
res.addChild(parseExpression(RANGE_MAX.get((RangeTable)tbl), false));
}
else
- assert0(false, "Unexpected Table implementation [cls=" + tbl.getClass().getSimpleName() + ", " +
- "sql=" + sql + ']');
+ assert0(false, "Unexpected Table implementation [cls=" + tbl.getClass().getSimpleName() + ']');
h2ObjToGridObj.put(tbl, res);
}
@@ -369,7 +396,7 @@ public class GridSqlQueryParser {
/**
* @param select Select.
*/
- public GridSqlSelect parse(Select select, @Nullable Query idxQry) {
+ private GridSqlSelect parseSelect(Select select) {
GridSqlSelect res = (GridSqlSelect)h2ObjToGridObj.get(select);
if (res != null)
@@ -384,26 +411,39 @@ public class GridSqlQueryParser {
Expression where = CONDITION.get(select);
res.where(parseExpression(where, false));
- GridSqlElement from = null;
+ ArrayList<TableFilter> tableFilters = new ArrayList<>();
TableFilter filter = select.getTopTableFilter();
- if (idxQry instanceof Select)
- filter = ((Select)idxQry).getTopTableFilter();
-
do {
assert0(filter != null, select);
assert0(filter.getNestedJoin() == null, select);
- GridSqlElement gridFilter = parseTable(filter);
+ // Can use optimized join order only if we are not inside of an expression.
+ if (parsingSubQryExpression == 0 && optimizedTableFilterOrder != null) {
+ String tblAlias = filter.getTableAlias();
+ int idx = optimizedTableFilterOrder.get(tblAlias);
- from = from == null ? gridFilter : new GridSqlJoin(from, gridFilter, filter.isJoinOuter(),
- parseExpression(filter.getJoinCondition(), false));
+ setElementAt(tableFilters, idx, filter);
+ }
+ else
+ tableFilters.add(filter);
filter = filter.getJoin();
}
while (filter != null);
+ // Build FROM clause from correctly ordered table filters.
+ GridSqlElement from = null;
+
+ for (int i = 0; i < tableFilters.size(); i++) {
+ TableFilter f = tableFilters.get(i);
+ GridSqlElement gridFilter = parseTable(f);
+
+ from = from == null ? gridFilter : new GridSqlJoin(from, gridFilter, f.isJoinOuter(),
+ parseExpression(f.getJoinCondition(), false));
+ }
+
res.from(from);
ArrayList<Expression> expressions = select.getExpressions();
@@ -430,6 +470,20 @@ public class GridSqlQueryParser {
}
/**
+ * @param list List.
+ * @param idx Index.
+ * @param x Element.
+ */
+ private static <Z> void setElementAt(List<Z> list, int idx, Z x) {
+ while (list.size() <= idx)
+ list.add(null);
+
+ assert0(list.get(idx) == null, "Element already set: " + idx);
+
+ list.set(idx, x);
+ }
+
+ /**
* @param merge Merge.
* @see <a href="http://h2database.com/html/grammar.html#merge">H2 merge spec</a>
*/
@@ -443,7 +497,7 @@ public class GridSqlQueryParser {
h2ObjToGridObj.put(merge, res);
Table srcTbl = MERGE_TABLE.get(merge);
- GridSqlElement tbl = parseTable(srcTbl, null, merge.getSQL());
+ GridSqlElement tbl = parseTable(srcTbl);
res.into(tbl);
@@ -452,7 +506,7 @@ public class GridSqlQueryParser {
GridSqlColumn[] cols = new GridSqlColumn[srcCols.length];
for (int i = 0; i < srcCols.length; i++) {
- cols[i] = new GridSqlColumn(srcCols[i], tbl, srcCols[i].getName(), srcCols[i].getSQL());
+ cols[i] = new GridSqlColumn(srcCols[i], tbl, null, null, srcCols[i].getName());
cols[i].resultType(fromColumn(srcCols[i]));
}
@@ -463,7 +517,7 @@ public class GridSqlQueryParser {
GridSqlColumn[] keys = new GridSqlColumn[srcKeys.length];
for (int i = 0; i < srcKeys.length; i++)
- keys[i] = new GridSqlColumn(srcKeys[i], tbl, srcKeys[i].getName(), srcKeys[i].getSQL());
+ keys[i] = new GridSqlColumn(srcKeys[i], tbl, null, null, srcKeys[i].getName());
res.keys(keys);
List<Expression[]> srcRows = MERGE_ROWS.get(merge);
@@ -481,7 +535,7 @@ public class GridSqlQueryParser {
}
else {
res.rows(Collections.<GridSqlElement[]>emptyList());
- res.query(parse(MERGE_QUERY.get(merge), null));
+ res.query(parseQuery(MERGE_QUERY.get(merge)));
}
return res;
@@ -501,7 +555,7 @@ public class GridSqlQueryParser {
h2ObjToGridObj.put(insert, res);
Table srcTbl = INSERT_TABLE.get(insert);
- GridSqlElement tbl = parseTable(srcTbl, null, insert.getSQL());
+ GridSqlElement tbl = parseTable(srcTbl);
res.into(tbl).
direct(INSERT_DIRECT.get(insert)).
@@ -511,7 +565,7 @@ public class GridSqlQueryParser {
GridSqlColumn[] cols = new GridSqlColumn[srcCols.length];
for (int i = 0; i < srcCols.length; i++) {
- cols[i] = new GridSqlColumn(srcCols[i], tbl, srcCols[i].getName(), srcCols[i].getSQL());
+ cols[i] = new GridSqlColumn(srcCols[i], tbl, null, null, srcCols[i].getName());
cols[i].resultType(fromColumn(srcCols[i]));
}
@@ -533,7 +587,7 @@ public class GridSqlQueryParser {
}
else {
res.rows(Collections.<GridSqlElement[]>emptyList());
- res.query(parse(INSERT_QUERY.get(insert), null));
+ res.query(parseQuery(INSERT_QUERY.get(insert)));
}
return res;
@@ -581,7 +635,7 @@ public class GridSqlQueryParser {
LinkedHashMap<String, GridSqlElement> set = new LinkedHashMap<>(srcSet.size());
for (Column c : srcCols) {
- GridSqlColumn col = new GridSqlColumn(c, tbl, c.getName(), c.getSQL());
+ GridSqlColumn col = new GridSqlColumn(c, tbl, null, null, c.getName());
col.resultType(fromColumn(c));
cols.add(col);
set.put(col.columnName(), parseExpression(srcSet.get(c), true));
@@ -631,65 +685,54 @@ public class GridSqlQueryParser {
}
/**
- * @param qry Prepared.
- * @return Query.
- */
- public GridSqlStatement parse(Prepared qry) {
- return parse(qry, null);
- }
-
- /**
- * @param qry Select.
+ * @param stmt Prepared statement.
+ * @return Parsed AST.
*/
- public GridSqlStatement parse(Prepared qry, @Nullable Query idxQry) {
- assert qry != null;
-
- if (qry instanceof Query)
- return parse((Query)qry, idxQry);
-
- if (qry instanceof Merge)
- return parseMerge((Merge)qry);
+ public final GridSqlStatement parse(Prepared stmt) {
+ if (stmt instanceof Query) {
+ if (optimizedTableFilterOrder != null)
+ collectOptimizedTableFiltersOrder((Query)stmt);
- if (qry instanceof Insert)
- return parseInsert((Insert)qry);
+ return parseQuery((Query)stmt);
+ }
- if (qry instanceof Delete)
- return parseDelete((Delete)qry);
+ if (stmt instanceof Merge)
+ return parseMerge((Merge)stmt);
- if (qry instanceof Update)
- return parseUpdate((Update)qry);
+ if (stmt instanceof Insert)
+ return parseInsert((Insert)stmt);
- if (qry instanceof Explain) {
- GridSqlStatement stmt = parse(EXPLAIN_COMMAND.get((Explain) qry));
+ if (stmt instanceof Delete)
+ return parseDelete((Delete)stmt);
- if (!(stmt instanceof GridSqlQuery))
- throw new IgniteSQLException("EXPLAIN is not supported for DML statement: " + qry,
- IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+ if (stmt instanceof Update)
+ return parseUpdate((Update)stmt);
- return stmt.explain(true);
- }
+ if (stmt instanceof Explain)
+ return parse(EXPLAIN_COMMAND.get((Explain)stmt)).explain(true);
- throw new IgniteSQLException("Unsupported statement: " + qry, IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+ throw new CacheException("Unsupported SQL statement: " + stmt);
}
/**
* @param qry Query.
- * @return Grid SQL query.
+ * @return Parsed query AST.
*/
- private GridSqlQuery parse(Query qry, @Nullable Query idxQry) {
+ private GridSqlQuery parseQuery(Query qry) {
if (qry instanceof Select)
- return parse((Select)qry, idxQry);
+ return parseSelect((Select)qry);
if (qry instanceof SelectUnion)
- return parse((SelectUnion)qry);
+ return parseUnion((SelectUnion)qry);
throw new UnsupportedOperationException("Unknown query type: " + qry);
}
/**
* @param union Select.
+ * @return Parsed AST.
*/
- public GridSqlUnion parse(SelectUnion union) {
+ private GridSqlUnion parseUnion(SelectUnion union) {
GridSqlUnion res = (GridSqlUnion)h2ObjToGridObj.get(union);
if (res != null)
@@ -697,8 +740,8 @@ public class GridSqlQueryParser {
res = new GridSqlUnion();
- res.right(parse(union.getRight(), null));
- res.left(parse(union.getLeft(), null));
+ res.right(parseQuery(union.getRight()));
+ res.left(parseQuery(union.getLeft()));
res.unionType(union.getUnionType());
@@ -736,6 +779,44 @@ public class GridSqlQueryParser {
}
/**
+ * @param qry Query.
+ */
+ private void collectOptimizedTableFiltersOrder(Query qry) {
+ if (qry instanceof SelectUnion) {
+ collectOptimizedTableFiltersOrder(((SelectUnion)qry).getLeft());
+ collectOptimizedTableFiltersOrder(((SelectUnion)qry).getRight());
+ }
+ else {
+ Select select = (Select)qry;
+
+ TableFilter filter = select.getTopTableFilter();
+
+ int i = 0;
+
+ do {
+ assert0(filter != null, select);
+ assert0(filter.getNestedJoin() == null, select);
+
+ // Here all the table filters must have generated unique aliases,
+ // thus we can store them in the same map for all the subqueries.
+ optimizedTableFilterOrder.put(filter.getTableAlias(), i++);
+
+ Table tbl = filter.getTable();
+
+ // Go down and collect inside of optimized subqueries.
+ if (tbl instanceof TableView) {
+ ViewIndex viewIdx = (ViewIndex)filter.getIndex();
+
+ collectOptimizedTableFiltersOrder(viewIdx.getQuery());
+ }
+
+ filter = filter.getJoin();
+ }
+ while (filter != null);
+ }
+ }
+
+ /**
* @param expression Expression.
* @param calcTypes Calculate types for all the expressions.
* @return Parsed expression.
@@ -746,8 +827,9 @@ public class GridSqlQueryParser {
return new GridSqlColumn(expCol.getColumn(),
parseTable(expCol.getTableFilter()),
- expression.getColumnName(),
- expression.getSQL());
+ SCHEMA_NAME.get(expCol),
+ expCol.getOriginalTableAliasName(),
+ expCol.getColumnName());
}
if (expression instanceof Alias)
@@ -812,9 +894,7 @@ public class GridSqlQueryParser {
if (expression instanceof Subquery) {
Query qry = ((Subquery)expression).getQuery();
- assert0(qry instanceof Select, expression);
-
- return new GridSqlSubquery(parse(qry, null));
+ return parseQueryExpression(qry);
}
if (expression instanceof ConditionIn) {
@@ -856,9 +936,7 @@ public class GridSqlQueryParser {
Query qry = QUERY.get((ConditionInSelect)expression);
- assert0(qry instanceof Select, qry);
-
- res.addChild(new GridSqlSubquery(parse(qry, null)));
+ res.addChild(parseQueryExpression(qry));
return res;
}
[37/50] [abbrv] ignite git commit: IGNITE-4741: Improved
JdbcStreamingSelfTest reliability. This closes #1568.
Posted by av...@apache.org.
IGNITE-4741: Improved JdbcStreamingSelfTest reliability. This closes #1568.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/19d6c31a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/19d6c31a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/19d6c31a
Branch: refs/heads/master
Commit: 19d6c31a24a1dc86aeec203a758d1d0d7387d1ed
Parents: 658b4ad
Author: Alexander Paschenko <al...@gmail.com>
Authored: Wed Feb 22 15:30:43 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Wed Feb 22 15:30:43 2017 +0300
----------------------------------------------------------------------
modules/clients/src/test/config/jdbc-config.xml | 2 +-
.../internal/jdbc2/JdbcStreamingSelfTest.java | 58 ++++++++++----------
2 files changed, 29 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/19d6c31a/modules/clients/src/test/config/jdbc-config.xml
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/config/jdbc-config.xml b/modules/clients/src/test/config/jdbc-config.xml
index 5e6e12c..27d2617 100644
--- a/modules/clients/src/test/config/jdbc-config.xml
+++ b/modules/clients/src/test/config/jdbc-config.xml
@@ -43,7 +43,7 @@
<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
<property name="addresses">
<list>
- <value>127.0.0.1:47500..47549</value>
+ <value>127.0.0.1:47500..47501</value>
</list>
</property>
</bean>
http://git-wip-us.apache.org/repos/asf/ignite/blob/19d6c31a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
index 5e206ee..6837a41 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcStreamingSelfTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.jdbc2;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
+import java.util.Collections;
import java.util.Properties;
import org.apache.ignite.IgniteJdbcDriver;
import org.apache.ignite.IgniteLogger;
@@ -28,7 +29,6 @@ import org.apache.ignite.configuration.ConnectorConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -40,9 +40,6 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
* Data streaming test.
*/
public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
- /** IP finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
/** JDBC URL. */
private static final String BASE_URL = CFG_URL_PREFIX + "modules/clients/src/test/config/jdbc-config.xml";
@@ -75,10 +72,14 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
);
cfg.setCacheConfiguration(cache);
+ cfg.setLocalHost("127.0.0.1");
TcpDiscoverySpi disco = new TcpDiscoverySpi();
- disco.setIpFinder(IP_FINDER);
+ TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+ ipFinder.setAddresses(Collections.singleton("127.0.0.1:47500..47501"));
+
+ disco.setIpFinder(ipFinder);
cfg.setDiscoverySpi(disco);
@@ -89,7 +90,7 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
- startGridsMultiThreaded(3);
+ startGrids(2);
Class.forName("org.apache.ignite.IgniteJdbcDriver");
}
@@ -131,29 +132,29 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
public void testStreamedInsert() throws Exception {
conn = createConnection(false);
- ignite(0).cache(null).put(5, 500);
+ for (int i = 10; i <= 100; i += 10)
+ ignite(0).cache(null).put(i, i * 100);
PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
- for (int i = 1; i <= 100000; i++) {
+ for (int i = 1; i <= 100; i++) {
stmt.setInt(1, i);
stmt.setInt(2, i);
stmt.executeUpdate();
}
- // Data is not there yet.
- assertNull(grid(0).cache(null).get(100000));
-
- // Let the stream flush.
- U.sleep(1500);
+ // Closing connection makes it wait for streamer close
+ // and thus for data load completion as well
+ conn.close();
// Now let's check it's all there.
- assertEquals(1, grid(0).cache(null).get(1));
- assertEquals(100000, grid(0).cache(null).get(100000));
-
- // 5 should still point to 500.
- assertEquals(500, grid(0).cache(null).get(5));
+ for (int i = 1; i <= 100; i++) {
+ if (i % 10 != 0)
+ assertEquals(i, grid(0).cache(null).get(i));
+ else // All that divides by 10 evenly should point to numbers 100 times greater - see above
+ assertEquals(i * 100, grid(0).cache(null).get(i));
+ }
}
/**
@@ -162,28 +163,25 @@ public class JdbcStreamingSelfTest extends GridCommonAbstractTest {
public void testStreamedInsertWithOverwritesAllowed() throws Exception {
conn = createConnection(true);
- ignite(0).cache(null).put(5, 500);
+ for (int i = 10; i <= 100; i += 10)
+ ignite(0).cache(null).put(i, i * 100);
PreparedStatement stmt = conn.prepareStatement("insert into Integer(_key, _val) values (?, ?)");
- for (int i = 1; i <= 100000; i++) {
+ for (int i = 1; i <= 100; i++) {
stmt.setInt(1, i);
stmt.setInt(2, i);
stmt.executeUpdate();
}
- // Data is not there yet.
- assertNull(grid(0).cache(null).get(100000));
-
- // Let the stream flush.
- U.sleep(1500);
+ // Closing connection makes it wait for streamer close
+ // and thus for data load completion as well
+ conn.close();
// Now let's check it's all there.
- assertEquals(1, grid(0).cache(null).get(1));
- assertEquals(100000, grid(0).cache(null).get(100000));
-
- // 5 should now point to 5 as we've turned overwriting on.
- assertEquals(5, grid(0).cache(null).get(5));
+ // i should point to i at all times as we've turned overwrites on above.
+ for (int i = 1; i <= 100; i++)
+ assertEquals(i, grid(0).cache(null).get(i));
}
}
[27/50] [abbrv] ignite git commit: Fix queries with binary marshaller.
Posted by av...@apache.org.
Fix queries with binary marshaller.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/de6e52ba
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/de6e52ba
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/de6e52ba
Branch: refs/heads/master
Commit: de6e52ba54ce58f0dadad16de8aae14b55ab8478
Parents: f4d36cb
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 16:27:08 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 16:27:08 2017 +0300
----------------------------------------------------------------------
.../internal/processors/query/h2/opt/GridH2TreeIndex.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/de6e52ba/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 07c3e6d..80597f2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -439,7 +439,11 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
assert ctx != null;
- Object o = ctx.toCacheKeyObject(row.getValue(KEY_COL));
+ final Value keyColValue = row.getValue(KEY_COL);
+
+ assert keyColValue != null;
+
+ final Object o = keyColValue.getObject();
if (o instanceof CacheObject)
key = (CacheObject)o;
[40/50] [abbrv] ignite git commit: IGNITE-3710 Upgrade ignite-spark
module to Spark 2.0 (cherry picked from commit b78d354)
Posted by av...@apache.org.
IGNITE-3710 Upgrade ignite-spark module to Spark 2.0
(cherry picked from commit b78d354)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e3d7326a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e3d7326a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e3d7326a
Branch: refs/heads/master
Commit: e3d7326ac8eb848666c87dcaf17361860dd5b73d
Parents: 94dab7b
Author: Evgenii Zhuravlev <ez...@gridgain.com>
Authored: Wed Feb 22 16:16:32 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Wed Feb 22 16:17:36 2017 +0300
----------------------------------------------------------------------
DEVNOTES.txt | 2 +-
examples/pom.xml | 14 ++
.../examples/java8/spark/SharedRDDExample.java | 110 ----------------
.../ignite/examples/spark/SharedRDDExample.java | 127 +++++++++++++++++++
.../ignite/examples/spark/package-info.java | 22 ++++
.../examples/SharedRDDExampleSelfTest.java | 36 ------
.../IgniteExamplesJ8SelfTestSuite.java | 2 -
.../examples/SharedRDDExampleSelfTest.java | 36 ++++++
.../IgniteExamplesSparkSelfTestSuite.java | 46 +++++++
9 files changed, 246 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/DEVNOTES.txt
----------------------------------------------------------------------
diff --git a/DEVNOTES.txt b/DEVNOTES.txt
index 6a275fc..79bedfd 100644
--- a/DEVNOTES.txt
+++ b/DEVNOTES.txt
@@ -33,7 +33,7 @@ Ignite Hadoop Accelerator Maven Build Instructions
mvn clean package -DskipTests -Dignite.edition=hadoop [-Dhadoop.version=X.X.X] [-Dspark.version=x.y.z]
Use 'hadoop.version' parameter to build Ignite against a specific Hadoop version.
-Use 'spark.version' parameter to build ignite-spark module for a specific Spark version.
+Use 'spark.version' parameter to build ignite-spark module for a specific Spark version. Version should be >= 2.0.0.
Look for apache-ignite-hadoop-<version>-bin.zip in ./target/bin directory. Resulting binary
assembly will also include integration module for Apache Spark.
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 1c4ad25..e7fd67d 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -112,6 +112,8 @@
<!-- will be changed by profile activation. allows to combine profiles. -->
<lgpl.folder>src/main/java</lgpl.folder>
<java8.folder>src/main/java</java8.folder>
+ <spark.folder>src/main/java</spark.folder>
+ <spark.test.folder>src/test/java</spark.test.folder>
<lgpl.test.folder>src/test/java</lgpl.test.folder>
<java8.test.folder>src/test/java</java8.test.folder>
</properties>
@@ -120,6 +122,11 @@
<profile>
<id>scala</id>
+ <properties>
+ <spark.folder>src/main/spark</spark.folder>
+ <spark.test.folder>src/test/spark</spark.test.folder>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ignite</groupId>
@@ -166,6 +173,11 @@
<profile>
<id>scala-2.10</id>
+ <properties>
+ <spark.folder>src/main/spark</spark.folder>
+ <spark.test.folder>src/test/spark</spark.test.folder>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ignite</groupId>
@@ -273,6 +285,7 @@
<source>schema-import/src/main/java</source>
<source>${lgpl.folder}</source>
<source>${java8.folder}</source>
+ <source>${spark.folder}</source>
</sources>
</configuration>
</execution>
@@ -286,6 +299,7 @@
<configuration>
<sources>
<source>${lgpl.test.folder}</source>
+ <source>${spark.test.folder}</source>
<source>${java8.test.folder}</source>
</sources>
</configuration>
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java b/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java
deleted file mode 100644
index 5f74a94..0000000
--- a/examples/src/main/java8/org/apache/ignite/examples/java8/spark/SharedRDDExample.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.java8.spark;
-
-import org.apache.ignite.spark.JavaIgniteContext;
-import org.apache.ignite.spark.JavaIgniteRDD;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.spark.SparkConf;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.PairFunction;
-import org.apache.spark.sql.Dataset;
-import scala.Tuple2;
-
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-/**
- * This example demonstrates how to create an JavaIgnitedRDD and share it with multiple spark workers. The goal of this
- * particular example is to provide the simplest code example of this logic.
- * <p>
- * This example will start Ignite in the embedded mode and will start an JavaIgniteContext on each Spark worker node.
- * <p>
- * The example can work in the standalone mode as well that can be enabled by setting JavaIgniteContext's
- * {@code standalone} property to {@code true} and running an Ignite node separately with
- * `examples/config/spark/example-shared-rdd.xml` config.
- */
-public class SharedRDDExample {
- /**
- * Executes the example.
- * @param args Command line arguments, none required.
- */
- public static void main(String args[]) {
- // Spark Configuration.
- SparkConf sparkConf = new SparkConf()
- .setAppName("JavaIgniteRDDExample")
- .setMaster("local")
- .set("spark.executor.instances", "2");
-
- // Spark context.
- JavaSparkContext sparkContext = new JavaSparkContext(sparkConf);
-
- // Adjust the logger to exclude the logs of no interest.
- Logger.getRootLogger().setLevel(Level.ERROR);
- Logger.getLogger("org.apache.ignite").setLevel(Level.INFO);
-
- // Creates Ignite context with specific configuration and runs Ignite in the embedded mode.
- JavaIgniteContext<Integer, Integer> igniteContext = new JavaIgniteContext<Integer, Integer>(
- sparkContext,"examples/config/spark/example-shared-rdd.xml", false);
-
- // Create a Java Ignite RDD of Type (Int,Int) Integer Pair.
- JavaIgniteRDD<Integer, Integer> sharedRDD = igniteContext.<Integer, Integer>fromCache("sharedRDD");
-
- // Define data to be stored in the Ignite RDD (cache).
- List<Integer> data = IntStream.range(0, 20).boxed().collect(Collectors.toList());
-
- // Preparing a Java RDD.
- JavaRDD<Integer> javaRDD = sparkContext.<Integer>parallelize(data);
-
- // Fill the Ignite RDD in with Int pairs. Here Pairs are represented as Scala Tuple2.
- sharedRDD.savePairs(javaRDD.<Integer, Integer>mapToPair(new PairFunction<Integer, Integer, Integer>() {
- @Override public Tuple2<Integer, Integer> call(Integer val) throws Exception {
- return new Tuple2<Integer, Integer>(val, val);
- }
- }));
-
- System.out.println(">>> Iterating over Ignite Shared RDD...");
-
- // Iterate over the Ignite RDD.
- sharedRDD.foreach((x) -> System.out.println("(" + x._1 + "," + x._2 + ")"));
-
- System.out.println(">>> Transforming values stored in Ignite Shared RDD...");
-
- // Filter out even values as a transformed RDD.
- JavaPairRDD<Integer, Integer> transformedValues =
- sharedRDD.filter((Tuple2<Integer, Integer> pair) -> pair._2() % 2 == 0);
-
- // Print out the transformed values.
- transformedValues.foreach((x) -> System.out.println("(" + x._1 + "," + x._2 + ")"));
-
- System.out.println(">>> Executing SQL query over Ignite Shared RDD...");
-
- // Execute SQL query over the Ignite RDD.
- Dataset df = sharedRDD.sql("select _val from Integer where _key < 9");
-
- // Show the result of the execution.
- df.show();
-
- // Close IgniteContext on all the workers.
- igniteContext.close(true);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java b/examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java
new file mode 100644
index 0000000..d8e5f16
--- /dev/null
+++ b/examples/src/main/spark/org/apache/ignite/examples/spark/SharedRDDExample.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.spark;
+
+import org.apache.ignite.spark.JavaIgniteContext;
+import org.apache.ignite.spark.JavaIgniteRDD;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.PairFunction;
+import org.apache.spark.api.java.function.VoidFunction;
+import org.apache.spark.sql.Dataset;
+import scala.Tuple2;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This example demonstrates how to create an JavaIgnitedRDD and share it with multiple spark workers. The goal of this
+ * particular example is to provide the simplest code example of this logic.
+ * <p>
+ * This example will start Ignite in the embedded mode and will start an JavaIgniteContext on each Spark worker node.
+ * <p>
+ * The example can work in the standalone mode as well that can be enabled by setting JavaIgniteContext's
+ * {@code standalone} property to {@code true} and running an Ignite node separately with
+ * `examples/config/spark/example-shared-rdd.xml` config.
+ */
+public class SharedRDDExample {
+ /**
+ * Executes the example.
+ * @param args Command line arguments, none required.
+ */
+ public static void main(String args[]) {
+ // Spark Configuration.
+ SparkConf sparkConf = new SparkConf()
+ .setAppName("JavaIgniteRDDExample")
+ .setMaster("local")
+ .set("spark.executor.instances", "2");
+
+ // Spark context.
+ JavaSparkContext sparkContext = new JavaSparkContext(sparkConf);
+
+ // Adjust the logger to exclude the logs of no interest.
+ Logger.getRootLogger().setLevel(Level.ERROR);
+ Logger.getLogger("org.apache.ignite").setLevel(Level.INFO);
+
+ // Creates Ignite context with specific configuration and runs Ignite in the embedded mode.
+ JavaIgniteContext<Integer, Integer> igniteContext = new JavaIgniteContext<Integer, Integer>(
+ sparkContext,"examples/config/spark/example-shared-rdd.xml", false);
+
+ // Create a Java Ignite RDD of Type (Int,Int) Integer Pair.
+ JavaIgniteRDD<Integer, Integer> sharedRDD = igniteContext.<Integer, Integer>fromCache("sharedRDD");
+
+ // Define data to be stored in the Ignite RDD (cache).
+ List<Integer> data = new ArrayList<>(20);
+
+ for (int i = 0; i<20; i++) {
+ data.add(i);
+ }
+
+ // Preparing a Java RDD.
+ JavaRDD<Integer> javaRDD = sparkContext.<Integer>parallelize(data);
+
+ // Fill the Ignite RDD in with Int pairs. Here Pairs are represented as Scala Tuple2.
+ sharedRDD.savePairs(javaRDD.<Integer, Integer>mapToPair(new PairFunction<Integer, Integer, Integer>() {
+ @Override public Tuple2<Integer, Integer> call(Integer val) throws Exception {
+ return new Tuple2<Integer, Integer>(val, val);
+ }
+ }));
+
+ System.out.println(">>> Iterating over Ignite Shared RDD...");
+
+ // Iterate over the Ignite RDD.
+ sharedRDD.foreach(new VoidFunction<Tuple2<Integer, Integer>>() {
+ @Override public void call(Tuple2<Integer, Integer> tuple) throws Exception {
+ System.out.println("(" + tuple._1 + "," + tuple._2 + ")");
+ }
+ });
+
+ System.out.println(">>> Transforming values stored in Ignite Shared RDD...");
+
+ // Filter out even values as a transformed RDD.
+ JavaPairRDD<Integer, Integer> transformedValues =
+ sharedRDD.filter(new Function<Tuple2<Integer, Integer>, Boolean>() {
+ @Override public Boolean call(Tuple2<Integer, Integer> tuple) throws Exception {
+ return tuple._2() % 2 == 0;
+ }
+ });
+
+ // Print out the transformed values.
+ transformedValues.foreach(new VoidFunction<Tuple2<Integer, Integer>>() {
+ @Override public void call(Tuple2<Integer, Integer> tuple) throws Exception {
+ System.out.println("(" + tuple._1 + "," + tuple._2 + ")");
+ }
+ });
+
+ System.out.println(">>> Executing SQL query over Ignite Shared RDD...");
+
+ // Execute SQL query over the Ignite RDD.
+ Dataset df = sharedRDD.sql("select _val from Integer where _key < 9");
+
+ // Show the result of the execution.
+ df.show();
+
+ // Close IgniteContext on all the workers.
+ igniteContext.close(true);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java b/examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java
new file mode 100644
index 0000000..f97a7b7
--- /dev/null
+++ b/examples/src/main/spark/org/apache/ignite/examples/spark/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Basic examples for ignite functionality with spark.
+ */
+package org.apache.ignite.examples.spark;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/src/test/java8/org/apache/ignite/java8/examples/SharedRDDExampleSelfTest.java
----------------------------------------------------------------------
diff --git a/examples/src/test/java8/org/apache/ignite/java8/examples/SharedRDDExampleSelfTest.java b/examples/src/test/java8/org/apache/ignite/java8/examples/SharedRDDExampleSelfTest.java
deleted file mode 100644
index 0fafb4d..0000000
--- a/examples/src/test/java8/org/apache/ignite/java8/examples/SharedRDDExampleSelfTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.java8.examples;
-
-import org.apache.ignite.examples.java8.spark.SharedRDDExample;
-import org.junit.Test;
-
-/**
- * SharedRDD examples self test.
- */
-public class SharedRDDExampleSelfTest {
- static final String[] EMPTY_ARGS = new String[0];
- /**
- * @throws Exception If failed.
- */
- @Test
- public void testSharedRDDExample() throws Exception {
- SharedRDDExample.main(EMPTY_ARGS);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java
----------------------------------------------------------------------
diff --git a/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java b/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java
index c32339f..7b62ea8 100644
--- a/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java
+++ b/examples/src/test/java8/org/apache/ignite/java8/testsuites/IgniteExamplesJ8SelfTestSuite.java
@@ -26,7 +26,6 @@ import org.apache.ignite.java8.examples.EventsExamplesMultiNodeSelfTest;
import org.apache.ignite.java8.examples.EventsExamplesSelfTest;
import org.apache.ignite.java8.examples.IndexingBridgeMethodTest;
import org.apache.ignite.java8.examples.MessagingExamplesSelfTest;
-import org.apache.ignite.java8.examples.SharedRDDExampleSelfTest;
import org.apache.ignite.testframework.GridTestUtils;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_OVERRIDE_MCAST_GRP;
@@ -50,7 +49,6 @@ public class IgniteExamplesJ8SelfTestSuite extends TestSuite {
suite.addTest(new TestSuite(IndexingBridgeMethodTest.class));
suite.addTest(new TestSuite(CacheExamplesSelfTest.class));
suite.addTest(new TestSuite(BasicExamplesSelfTest.class));
- suite.addTest(new TestSuite(SharedRDDExampleSelfTest.class));
// suite.addTest(new TestSuite(ContinuationExamplesSelfTest.class));
// suite.addTest(new TestSuite(ContinuousMapperExamplesSelfTest.class));
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java
----------------------------------------------------------------------
diff --git a/examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java b/examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java
new file mode 100644
index 0000000..038202e
--- /dev/null
+++ b/examples/src/test/spark/org/apache/ignite/spark/examples/SharedRDDExampleSelfTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.examples;
+
+import org.apache.ignite.examples.spark.SharedRDDExample;
+import org.junit.Test;
+
+/**
+ * SharedRDD examples self test.
+ */
+public class SharedRDDExampleSelfTest {
+ static final String[] EMPTY_ARGS = new String[0];
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testSharedRDDExample() throws Exception {
+ SharedRDDExample.main(EMPTY_ARGS);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d7326a/examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java b/examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java
new file mode 100644
index 0000000..73b286a
--- /dev/null
+++ b/examples/src/test/spark/org/apache/ignite/spark/testsuites/IgniteExamplesSparkSelfTestSuite.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.spark.examples.SharedRDDExampleSelfTest;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_OVERRIDE_MCAST_GRP;
+
+/**
+ * Examples test suite.
+ * <p>
+ * Contains only Spring ignite examples tests.
+ */
+public class IgniteExamplesSparkSelfTestSuite extends TestSuite {
+ /**
+ * @return Suite.
+ * @throws Exception If failed.
+ */
+ public static TestSuite suite() throws Exception {
+ System.setProperty(IGNITE_OVERRIDE_MCAST_GRP,
+ GridTestUtils.getNextMulticastGroup(IgniteExamplesSparkSelfTestSuite.class));
+
+ TestSuite suite = new TestSuite("Ignite Examples Test Suite");
+
+ suite.addTest(new TestSuite(SharedRDDExampleSelfTest.class));
+
+ return suite;
+ }
+}
[33/50] [abbrv] ignite git commit: Merge branch 'ignite-1.9' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-1.9
Posted by av...@apache.org.
Merge branch 'ignite-1.9' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-1.9
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/437f3746
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/437f3746
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/437f3746
Branch: refs/heads/master
Commit: 437f37469e241b30aa4dfd129ba572440f487161
Parents: efb8668 a160162
Author: Sergi Vladykin <se...@gmail.com>
Authored: Wed Feb 22 02:32:07 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Wed Feb 22 02:32:07 2017 +0300
----------------------------------------------------------------------
assembly/release-base.xml | 5 -----
assembly/release-fabric-base.xml | 5 +++++
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[09/50] [abbrv] ignite git commit: ignite-3860 - merge to 1.9 #
Conflicts:
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
# modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlTable.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlTable.java
index 49c679d..57ca4df 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlTable.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlTable.java
@@ -36,9 +36,6 @@ public class GridSqlTable extends GridSqlElement {
/** */
private final GridH2Table tbl;
- /** */
- private boolean affKeyCond;
-
/**
* @param schema Schema.
* @param tblName Table name.
@@ -60,7 +57,7 @@ public class GridSqlTable extends GridSqlElement {
* @param tbl H2 Table.
*/
private GridSqlTable(@Nullable String schema, String tblName, @Nullable Table tbl) {
- super(Collections.<GridSqlElement>emptyList());
+ super(Collections.<GridSqlAst>emptyList());
assert schema != null : "schema";
assert tblName != null : "tblName";
@@ -71,20 +68,6 @@ public class GridSqlTable extends GridSqlElement {
this.tbl = tbl instanceof GridH2Table ? (GridH2Table)tbl : null;
}
- /**
- * @param affKeyCond If affinity key condition is found.
- */
- public void affinityKeyCondition(boolean affKeyCond) {
- this.affKeyCond = affKeyCond;
- }
-
- /**
- * @return {@code true} If affinity key condition is found.
- */
- public boolean affinityKeyCondition() {
- return affKeyCond;
- }
-
/** {@inheritDoc} */
@Override public String getSQL() {
if (schema == null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlType.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlType.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlType.java
index efe9138..b4a610c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlType.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlType.java
@@ -48,6 +48,10 @@ public final class GridSqlType {
public static final GridSqlType BOOLEAN = new GridSqlType(Value.BOOLEAN, 0, ValueBoolean.PRECISION,
ValueBoolean.DISPLAY_SIZE, "BOOLEAN");
+ /** */
+ public static final GridSqlType RESULT_SET = new GridSqlType(Value.RESULT_SET, 0,
+ Integer.MAX_VALUE, Integer.MAX_VALUE, "");
+
/** H2 type. */
private final int type;
@@ -71,7 +75,7 @@ public final class GridSqlType {
* @param sql SQL definition of the type.
*/
private GridSqlType(int type, int scale, long precision, int displaySize, String sql) {
- assert !F.isEmpty(sql) || type == Value.UNKNOWN;
+ assert !F.isEmpty(sql) || type == Value.UNKNOWN || type == Value.RESULT_SET;
this.type = type;
this.scale = scale;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java
index b11278e..09f0e24 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlUnion.java
@@ -26,6 +26,12 @@ import org.h2.util.StatementBuilder;
*/
public class GridSqlUnion extends GridSqlQuery {
/** */
+ public static final int LEFT_CHILD = 2;
+
+ /** */
+ public static final int RIGHT_CHILD = 3;
+
+ /** */
private int unionType;
/** */
@@ -35,6 +41,57 @@ public class GridSqlUnion extends GridSqlQuery {
private GridSqlQuery left;
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public <E extends GridSqlAst> E child(int childIdx) {
+ if (childIdx < LEFT_CHILD)
+ return super.child(childIdx);
+
+ switch (childIdx) {
+ case LEFT_CHILD:
+ assert left != null;
+
+ return (E)left;
+
+ case RIGHT_CHILD:
+ assert right != null;
+
+ return (E)right;
+
+ default:
+ throw new IllegalStateException("Child index: " + childIdx);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public <E extends GridSqlAst> void child(int childIdx, E child) {
+ if (childIdx < LEFT_CHILD) {
+ super.child(childIdx, child);
+
+ return;
+ }
+
+ switch (childIdx) {
+ case LEFT_CHILD:
+ left = (GridSqlQuery)child;
+
+ break;
+
+ case RIGHT_CHILD:
+ right = (GridSqlQuery)child;
+
+ break;
+
+ default:
+ throw new IllegalStateException("Child index: " + childIdx);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int size() {
+ return 4; // OFFSET + LIMIT + LEFT + RIGHT
+ }
+
+ /** {@inheritDoc} */
@Override protected int visibleColumns() {
return left.visibleColumns();
}
@@ -50,7 +107,7 @@ public class GridSqlUnion extends GridSqlQuery {
buff.append('(').append(left.getSQL()).append(')');
- switch (unionType) {
+ switch (unionType()) {
case SelectUnion.UNION_ALL:
buff.append("\nUNION ALL\n");
break;
@@ -78,6 +135,13 @@ public class GridSqlUnion extends GridSqlQuery {
return buff.toString();
}
+ /** {@inheritDoc} */
+ @Override public boolean simpleQuery() {
+ return unionType() == SelectUnion.UNION_ALL && sort().isEmpty() &&
+ offset() == null && limit() == null &&
+ left().simpleQuery() && right().simpleQuery();
+ }
+
/**
* @return Union type.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlValue.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlValue.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlValue.java
deleted file mode 100644
index 665268c..0000000
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlValue.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.query.h2.sql;
-
-/**
- * Marker interface for a simple value.
- */
-public interface GridSqlValue {
- // No-op.
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index ac1a6a6..b0fa639 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -84,6 +84,7 @@ import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.setupConnection;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.MAP;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REPLICATED;
import static org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.QUERY_POOL;
@@ -435,6 +436,7 @@ public class GridMapQueryExecutor {
null,
req.pageSize(),
false,
+ true,
req.timeout());
}
@@ -456,6 +458,7 @@ public class GridMapQueryExecutor {
req.tables(),
req.pageSize(),
req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS),
+ req.isFlagSet(GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER),
req.timeout());
}
@@ -482,6 +485,7 @@ public class GridMapQueryExecutor {
Collection<String> tbls,
int pageSize,
boolean distributedJoins,
+ boolean enforceJoinOrder,
int timeout
) {
// Prepare to run queries.
@@ -541,8 +545,7 @@ public class GridMapQueryExecutor {
Connection conn = h2.connectionForSpace(mainCctx.name());
- // Here we enforce join order to have the same behavior on all the nodes.
- h2.setupConnection(conn, distributedJoins, true);
+ setupConnection(conn, distributedJoins, enforceJoinOrder);
GridH2QueryContext.set(qctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
index c267f4a..de14771 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
@@ -17,21 +17,25 @@
package org.apache.ignite.internal.processors.query.h2.twostep;
+import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.ConcurrentModificationException;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.RandomAccess;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import javax.cache.CacheException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.processors.query.h2.opt.GridH2Cursor;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.h2.engine.Session;
import org.h2.index.BaseIndex;
@@ -43,9 +47,13 @@ import org.h2.result.SearchRow;
import org.h2.result.SortOrder;
import org.h2.table.IndexColumn;
import org.h2.table.TableFilter;
+import org.h2.value.Value;
import org.jetbrains.annotations.Nullable;
+import static java.util.Collections.emptyIterator;
+import static java.util.Objects.requireNonNull;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_MERGE_TABLE_MAX_SIZE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
/**
@@ -55,6 +63,27 @@ public abstract class GridMergeIndex extends BaseIndex {
/** */
private static final int MAX_FETCH_SIZE = getInteger(IGNITE_SQL_MERGE_TABLE_MAX_SIZE, 10_000);
+ /** */
+ private static final int PREFETCH_SIZE = getInteger(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE, 1024);
+
+ /** */
+ protected final Comparator<SearchRow> firstRowCmp = new Comparator<SearchRow>() {
+ @Override public int compare(SearchRow rowInList, SearchRow searchRow) {
+ int res = compareRows(rowInList, searchRow);
+
+ return res == 0 ? 1 : res;
+ }
+ };
+
+ /** */
+ protected final Comparator<SearchRow> lastRowCmp = new Comparator<SearchRow>() {
+ @Override public int compare(SearchRow rowInList, SearchRow searchRow) {
+ int res = compareRows(rowInList, searchRow);
+
+ return res == 0 ? -1 : res;
+ }
+ };
+
/** All rows number. */
private final AtomicInteger expRowsCnt = new AtomicInteger(0);
@@ -67,10 +96,13 @@ public abstract class GridMergeIndex extends BaseIndex {
/**
* Will be r/w from query execution thread only, does not need to be threadsafe.
*/
- private ArrayList<Row> fetched = new ArrayList<>();
+ private final BlockList<Row> fetched;
/** */
- private int fetchedCnt;
+ private Row lastEvictedRow;
+
+ /** */
+ private volatile int fetchedCnt;
/** */
private final GridKernalContext ctx;
@@ -86,8 +118,9 @@ public abstract class GridMergeIndex extends BaseIndex {
GridMergeTable tbl,
String name,
IndexType type,
- IndexColumn[] cols) {
- this.ctx = ctx;
+ IndexColumn[] cols
+ ) {
+ this(ctx);
initBaseIndex(tbl, 0, name, cols, type);
}
@@ -96,7 +129,19 @@ public abstract class GridMergeIndex extends BaseIndex {
* @param ctx Context.
*/
protected GridMergeIndex(GridKernalContext ctx) {
+ if (!U.isPow2(PREFETCH_SIZE)) {
+ throw new IllegalArgumentException(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE + " (" + PREFETCH_SIZE +
+ ") must be positive and a power of 2.");
+ }
+
+ if (PREFETCH_SIZE >= MAX_FETCH_SIZE) {
+ throw new IllegalArgumentException(IGNITE_SQL_MERGE_TABLE_PREFETCH_SIZE + " (" + PREFETCH_SIZE +
+ ") must be less than " + IGNITE_SQL_MERGE_TABLE_MAX_SIZE + " (" + MAX_FETCH_SIZE + ").");
+ }
+
this.ctx = ctx;
+
+ fetched = new BlockList<>(PREFETCH_SIZE);
}
/**
@@ -109,7 +154,7 @@ public abstract class GridMergeIndex extends BaseIndex {
/**
* Fails index if any source node is left.
*/
- protected final void checkSourceNodesAlive() {
+ private void checkSourceNodesAlive() {
for (UUID nodeId : sources()) {
if (!ctx.discovery().alive(nodeId)) {
fail(nodeId, null);
@@ -154,6 +199,50 @@ public abstract class GridMergeIndex extends BaseIndex {
}
/**
+ * @param queue Queue to poll.
+ * @return Next page.
+ */
+ private GridResultPage takeNextPage(BlockingQueue<GridResultPage> queue) {
+ GridResultPage page;
+
+ for (;;) {
+ try {
+ page = queue.poll(500, TimeUnit.MILLISECONDS);
+ }
+ catch (InterruptedException e) {
+ throw new CacheException("Query execution was interrupted.", e);
+ }
+
+ if (page != null)
+ break;
+
+ checkSourceNodesAlive();
+ }
+
+ return page;
+ }
+
+ /**
+ * @param queue Queue to poll.
+ * @param iter Current iterator.
+ * @return The same or new iterator.
+ */
+ protected final Iterator<Value[]> pollNextIterator(BlockingQueue<GridResultPage> queue, Iterator<Value[]> iter) {
+ while (!iter.hasNext()) {
+ GridResultPage page = takeNextPage(queue);
+
+ if (page.isLast())
+ return emptyIterator(); // We are done.
+
+ fetchNextPage(page);
+
+ iter = page.rows();
+ }
+
+ return iter;
+ }
+
+ /**
* @param e Error.
*/
public void fail(final CacheException e) {
@@ -261,9 +350,8 @@ public abstract class GridMergeIndex extends BaseIndex {
}
/** {@inheritDoc} */
- @Override public Cursor find(Session ses, SearchRow first, SearchRow last) {
- if (fetched == null)
- throw new IgniteException("Fetched result set was too large.");
+ @Override public final Cursor find(Session ses, SearchRow first, SearchRow last) {
+ checkBounds(lastEvictedRow, first, last);
if (fetchedAll())
return findAllFetched(fetched, first, last);
@@ -279,16 +367,26 @@ public abstract class GridMergeIndex extends BaseIndex {
}
/**
- * @param first First row.
- * @param last Last row.
+ * @param lastEvictedRow Last evicted fetched row.
+ * @param first Lower bound.
+ * @param last Upper bound.
+ */
+ protected void checkBounds(Row lastEvictedRow, SearchRow first, SearchRow last) {
+ if (lastEvictedRow != null)
+ throw new IgniteException("Fetched result set was too large.");
+ }
+
+ /**
+ * @param first Lower bound.
+ * @param last Upper bound.
* @return Cursor. Usually it must be {@link FetchingCursor} instance.
*/
protected abstract Cursor findInStream(@Nullable SearchRow first, @Nullable SearchRow last);
/**
* @param fetched Fetched rows.
- * @param first First row.
- * @param last Last row.
+ * @param first Lower bound.
+ * @param last Upper bound.
* @return Cursor.
*/
protected abstract Cursor findAllFetched(List<Row> fetched, @Nullable SearchRow first, @Nullable SearchRow last);
@@ -349,69 +447,212 @@ public abstract class GridMergeIndex extends BaseIndex {
}
/**
+ * @param rows Sorted rows list.
+ * @param searchRow Search row.
+ * @param cmp Comparator.
+ * @param checkLast If we need to optimistically check the last row right away.
+ * @return Insertion point for the search row.
+ */
+ protected static int binarySearchRow(
+ List<Row> rows,
+ SearchRow searchRow,
+ Comparator<SearchRow> cmp,
+ boolean checkLast
+ ) {
+ assert !rows.isEmpty();
+
+ // Optimistically compare with the last row as a first step.
+ if (checkLast) {
+ int res = cmp.compare(last(rows), searchRow);
+
+ assert res != 0; // Comparators must never return 0 here.
+
+ if (res < 0)
+ return rows.size(); // The search row is greater than the last row.
+ }
+
+ int res = Collections.binarySearch(rows, searchRow, cmp);
+
+ assert res < 0: res; // Comparator must never return 0.
+
+ return -res - 1;
+ }
+
+ /**
+ * @param evictedBlock Evicted block.
+ */
+ private void onBlockEvict(List<Row> evictedBlock) {
+ assert evictedBlock.size() == PREFETCH_SIZE;
+
+ // Remember the last row (it will be max row) from the evicted block.
+ lastEvictedRow = requireNonNull(last(evictedBlock));
+ }
+
+ /**
+ * @param l List.
+ * @return Last element.
+ */
+ private static <Z> Z last(List<Z> l) {
+ return l.get(l.size() - 1);
+ }
+
+ /**
* Fetching cursor.
*/
- protected class FetchingCursor extends GridH2Cursor {
+ protected class FetchingCursor implements Cursor {
/** */
- private Iterator<Row> stream;
+ Iterator<Row> stream;
+
+ /** */
+ List<Row> rows;
+
+ /** */
+ int cur;
+
+ /** */
+ SearchRow first;
+
+ /** */
+ SearchRow last;
+
+ /** */
+ int lastFound = Integer.MAX_VALUE;
/**
- * @param stream Iterator.
+ * @param first Lower bound.
+ * @param last Upper bound.
+ * @param stream Stream of all the rows from remote nodes.
*/
- public FetchingCursor(Iterator<Row> stream) {
- super(new FetchedIterator());
-
+ public FetchingCursor(SearchRow first, SearchRow last, Iterator<Row> stream) {
assert stream != null;
+ // Initially we will use all the fetched rows, after we will switch to the last block.
+ rows = fetched;
+
this.stream = stream;
+ this.first = first;
+ this.last = last;
+
+ if (haveBounds() && !rows.isEmpty())
+ cur = findBounds();
+
+ cur--; // Set current position before the first row.
}
- /** {@inheritDoc} */
- @Override public boolean next() {
- if (super.next()) {
- assert cur != null;
-
- if (iter == stream && fetched != null) { // Cache fetched rows for reuse.
- if (fetched.size() == MAX_FETCH_SIZE)
- fetched = null; // Throw away fetched result if it is too large.
- else
- fetched.add(cur);
- }
+ /**
+ * @return {@code true} If we have bounds.
+ */
+ private boolean haveBounds() {
+ return first != null || last != null;
+ }
- fetchedCnt++;
+ /**
+ * @return Lower bound.
+ */
+ private int findBounds() {
+ assert !rows.isEmpty(): "rows";
- return true;
+ int firstFound = cur;
+
+ // Find the lower bound.
+ if (first != null) {
+ firstFound = binarySearchRow(rows, first, firstRowCmp, true);
+
+ assert firstFound >= cur && firstFound <= rows.size(): "firstFound";
+
+ if (firstFound == rows.size())
+ return firstFound; // The lower bound is greater than all the rows we have.
+
+ first = null; // We have found the lower bound, do not need it anymore.
}
- if (iter == stream) // We've fetched the stream.
- return false;
+ // Find the upper bound.
+ if (last != null) {
+ assert lastFound == Integer.MAX_VALUE: "lastFound";
+
+ int lastFound0 = binarySearchRow(rows, last, lastRowCmp, true);
- iter = stream; // Switch from cached to stream.
+ // If the upper bound is too large we will ignore it.
+ if (lastFound0 != rows.size())
+ lastFound = lastFound0;
+ }
- return next();
+ return firstFound;
}
- }
- /**
- * List iterator without {@link ConcurrentModificationException}.
- */
- private class FetchedIterator implements Iterator<Row> {
- /** */
- private int idx;
+ /**
+ * Fetch rows from the stream.
+ */
+ private void fetchRows() {
+ for (;;) {
+ // Take the current last block and set the position after last.
+ rows = fetched.lastBlock();
+ cur = rows.size();
+
+ // Fetch stream.
+ while (stream.hasNext()) {
+ fetched.add(requireNonNull(stream.next()));
+
+ // Evict block if we've fetched too many rows.
+ if (fetched.size() == MAX_FETCH_SIZE) {
+ onBlockEvict(fetched.evictFirstBlock());
+
+ assert fetched.size() < MAX_FETCH_SIZE;
+ }
+
+ // No bounds -> no need to do binary search, can return the fetched row right away.
+ if (!haveBounds())
+ break;
+
+ // When the last block changed, it means that we've filled the current last block.
+ // We have fetched the needed number of rows for binary search.
+ if (fetched.lastBlock() != rows) {
+ assert fetched.lastBlock().isEmpty(); // The last row must be added to the previous block.
+
+ break;
+ }
+ }
+
+ if (cur == rows.size())
+ cur = Integer.MAX_VALUE; // We were not able to fetch anything. Done.
+ else {
+ // Update fetched count.
+ fetchedCnt += rows.size() - cur;
+
+ if (haveBounds()) {
+ cur = findBounds();
+
+ if (cur == rows.size())
+ continue; // The lower bound is too large, continue fetching rows.
+ }
+ }
+
+ return;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean next() {
+ if (++cur == rows.size())
+ fetchRows();
+
+ return cur < lastFound;
+ }
/** {@inheritDoc} */
- @Override public boolean hasNext() {
- return fetched != null && idx < fetched.size();
+ @Override public Row get() {
+ return rows.get(cur);
}
/** {@inheritDoc} */
- @Override public Row next() {
- return fetched.get(idx++);
+ @Override public SearchRow getSearchRow() {
+ return get();
}
/** {@inheritDoc} */
- @Override public void remove() {
- throw new UnsupportedOperationException();
+ @Override public boolean previous() {
+ // Should never be called.
+ throw DbException.getUnsupportedException("previous");
}
}
@@ -427,4 +668,81 @@ public abstract class GridMergeIndex extends BaseIndex {
/** */
volatile State state = State.UNINITIALIZED;
}
+
+ /**
+ */
+ private static final class BlockList<Z> extends AbstractList<Z> implements RandomAccess {
+ /** */
+ private final List<List<Z>> blocks;
+
+ /** */
+ private int size;
+
+ /** */
+ private final int maxBlockSize;
+
+ /** */
+ private final int shift;
+
+ /** */
+ private final int mask;
+
+ /**
+ * @param maxBlockSize Max block size.
+ */
+ private BlockList(int maxBlockSize) {
+ assert U.isPow2(maxBlockSize);
+
+ this.maxBlockSize = maxBlockSize;
+
+ shift = Integer.numberOfTrailingZeros(maxBlockSize);
+ mask = maxBlockSize - 1;
+
+ blocks = new ArrayList<>();
+ blocks.add(new ArrayList<Z>());
+ }
+
+ /** {@inheritDoc} */
+ @Override public int size() {
+ return size;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean add(Z z) {
+ size++;
+
+ List<Z> lastBlock = lastBlock();
+
+ lastBlock.add(z);
+
+ if (lastBlock.size() == maxBlockSize)
+ blocks.add(new ArrayList<Z>());
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Z get(int idx) {
+ return blocks.get(idx >>> shift).get(idx & mask);
+ }
+
+ /**
+ * @return Last block.
+ */
+ private List<Z> lastBlock() {
+ return last(blocks);
+ }
+
+ /**
+ * @return Evicted block.
+ */
+ private List<Z> evictFirstBlock() {
+ // Remove head block.
+ List<Z> res = blocks.remove(0);
+
+ size -= res.size();
+
+ return res;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
new file mode 100644
index 0000000..a1b6691
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2Cursor;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.h2.index.Cursor;
+import org.h2.index.IndexType;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.table.IndexColumn;
+import org.h2.value.Value;
+import org.jetbrains.annotations.Nullable;
+
+import static java.util.Collections.emptyIterator;
+import static org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase.bubbleUp;
+
+/**
+ * Sorted index.
+ */
+public final class GridMergeIndexSorted extends GridMergeIndex {
+ /** */
+ private final Comparator<RowStream> streamCmp = new Comparator<RowStream>() {
+ @Override public int compare(RowStream o1, RowStream o2) {
+ // Nulls at the beginning.
+ if (o1 == null)
+ return -1;
+
+ if (o2 == null)
+ return 1;
+
+ return compareRows(o1.get(), o2.get());
+ }
+ };
+
+ /** */
+ private Map<UUID,RowStream> streamsMap;
+
+ /** */
+ private RowStream[] streams;
+
+ /**
+ * @param ctx Kernal context.
+ * @param tbl Table.
+ * @param name Index name,
+ * @param type Index type.
+ * @param cols Columns.
+ */
+ public GridMergeIndexSorted(
+ GridKernalContext ctx,
+ GridMergeTable tbl,
+ String name,
+ IndexType type,
+ IndexColumn[] cols
+ ) {
+ super(ctx, tbl, name, type, cols);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void setSources(Collection<ClusterNode> nodes) {
+ super.setSources(nodes);
+
+ streamsMap = U.newHashMap(nodes.size());
+ streams = new RowStream[nodes.size()];
+
+ int i = 0;
+
+ for (ClusterNode node : nodes) {
+ RowStream stream = new RowStream(node.id());
+
+ streams[i] = stream;
+
+ if (streamsMap.put(stream.src, stream) != null)
+ throw new IllegalStateException();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void addPage0(GridResultPage page) {
+ if (page.isLast() || page.isFail()) {
+ // Finish all the streams.
+ for (RowStream stream : streams)
+ stream.addPage(page);
+ }
+ else {
+ assert page.rowsInPage() > 0;
+
+ UUID src = page.source();
+
+ streamsMap.get(src).addPage(page);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void checkBounds(Row lastEvictedRow, SearchRow first, SearchRow last) {
+ // If our last evicted fetched row was smaller than the given lower bound,
+ // then we are ok. This is important for merge join to work.
+ if (lastEvictedRow != null && first != null && compareRows(lastEvictedRow, first) < 0)
+ return;
+
+ super.checkBounds(lastEvictedRow, first, last);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Cursor findAllFetched(List<Row> fetched, SearchRow first, SearchRow last) {
+ Iterator<Row> iter;
+
+ if (fetched.isEmpty())
+ iter = emptyIterator();
+ else if (first == null && last == null)
+ iter = fetched.iterator();
+ else {
+ int low = first == null ? 0 : binarySearchRow(fetched, first, firstRowCmp, false);
+
+ if (low == fetched.size())
+ iter = emptyIterator();
+ else {
+ int high = last == null ? fetched.size() : binarySearchRow(fetched, last, lastRowCmp, false);
+
+ iter = fetched.subList(low, high).iterator();
+ }
+ }
+
+ return new GridH2Cursor(iter);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Cursor findInStream(@Nullable SearchRow first, @Nullable SearchRow last) {
+ return new FetchingCursor(first, last, new MergeStreamIterator());
+ }
+
+ /**
+ * Iterator merging multiple row streams.
+ */
+ private final class MergeStreamIterator implements Iterator<Row> {
+ /** */
+ private boolean first = true;
+
+ /** */
+ private int off;
+
+ /** */
+ private boolean hasNext;
+
+ /**
+ *
+ */
+ private void goFirst() {
+ for (int i = 0; i < streams.length; i++) {
+ if (!streams[i].next()) {
+ streams[i] = null;
+ off++; // Move left bound.
+ }
+ }
+
+ if (off < streams.length)
+ Arrays.sort(streams, streamCmp);
+
+ first = false;
+ }
+
+ /**
+ *
+ */
+ private void goNext() {
+ if (streams[off].next())
+ bubbleUp(streams, off, streamCmp);
+ else
+ streams[off++] = null; // Move left bound and nullify empty stream.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasNext() {
+ if (hasNext)
+ return true;
+
+ if (first)
+ goFirst();
+ else
+ goNext();
+
+ return hasNext = off < streams.length;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Row next() {
+ if (!hasNext())
+ throw new NoSuchElementException();
+
+ hasNext = false;
+
+ return streams[off].get();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * Row stream.
+ */
+ private final class RowStream {
+ /** */
+ final UUID src;
+
+ /** */
+ final BlockingQueue<GridResultPage> queue = new ArrayBlockingQueue<>(8);
+
+ /** */
+ Iterator<Value[]> iter = emptyIterator();
+
+ /** */
+ Row cur;
+
+ /**
+ * @param src Source.
+ */
+ private RowStream(UUID src) {
+ this.src = src;
+ }
+
+ /**
+ * @param page Page.
+ */
+ private void addPage(GridResultPage page) {
+ queue.offer(page);
+ }
+
+ /**
+ * @return {@code true} If we successfully switched to the next row.
+ */
+ private boolean next() {
+ cur = null;
+
+ iter = pollNextIterator(queue, iter);
+
+ if (!iter.hasNext())
+ return false;
+
+ cur = GridH2RowFactory.create(iter.next());
+
+ return true;
+ }
+
+ /**
+ * @return Current row.
+ */
+ private Row get() {
+ assert cur != null;
+
+ return cur;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java
index 8a8577f..b69c898 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexUnsorted.java
@@ -22,8 +22,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import javax.cache.CacheException;
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Cursor;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowFactory;
@@ -33,12 +31,11 @@ import org.h2.result.Row;
import org.h2.result.SearchRow;
import org.h2.table.IndexColumn;
import org.h2.value.Value;
-import org.jetbrains.annotations.Nullable;
/**
* Unsorted merge index.
*/
-public class GridMergeIndexUnsorted extends GridMergeIndex {
+public final class GridMergeIndexUnsorted extends GridMergeIndex {
/** */
private final BlockingQueue<GridResultPage> queue = new LinkedBlockingQueue<>();
@@ -74,43 +71,22 @@ public class GridMergeIndexUnsorted extends GridMergeIndex {
}
/** {@inheritDoc} */
- @Override protected Cursor findAllFetched(List<Row> fetched, @Nullable SearchRow first, @Nullable SearchRow last) {
+ @Override protected Cursor findAllFetched(List<Row> fetched, SearchRow first, SearchRow last) {
+ // This index is unsorted: have to ignore bounds.
return new GridH2Cursor(fetched.iterator());
}
/** {@inheritDoc} */
- @Override protected Cursor findInStream(@Nullable SearchRow first, @Nullable SearchRow last) {
- return new FetchingCursor(new Iterator<Row>() {
+ @Override protected Cursor findInStream(SearchRow first, SearchRow last) {
+ // This index is unsorted: have to ignore bounds.
+ return new FetchingCursor(null, null, new Iterator<Row>() {
/** */
Iterator<Value[]> iter = Collections.emptyIterator();
@Override public boolean hasNext() {
- while (!iter.hasNext()) {
- GridResultPage page;
+ iter = pollNextIterator(queue, iter);
- for (;;) {
- try {
- page = queue.poll(500, TimeUnit.MILLISECONDS);
- }
- catch (InterruptedException e) {
- throw new CacheException("Query execution was interrupted.", e);
- }
-
- if (page != null)
- break;
-
- checkSourceNodesAlive();
- }
-
- if (page.isLast())
- return false; // We are done.
-
- fetchNextPage(page);
-
- iter = page.rows();
- }
-
- return true;
+ return iter.hasNext();
}
@Override public Row next() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index f54fab6..128ca8e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -67,7 +67,6 @@ import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator
import org.apache.ignite.internal.processors.query.GridQueryCancel;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext;
-import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlType;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse;
@@ -101,6 +100,7 @@ import org.jsr166.ConcurrentHashMap8;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
+import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter.mergeTableIdentifier;
/**
* Reduce query executor.
@@ -589,7 +589,8 @@ public class GridReduceQueryExecutor {
mapQrys = new ArrayList<>(qry.mapQueries().size());
for (GridCacheSqlQuery mapQry : qry.mapQueries())
- mapQrys.add(new GridCacheSqlQuery("EXPLAIN " + mapQry.query(), mapQry.parameters()));
+ mapQrys.add(new GridCacheSqlQuery("EXPLAIN " + mapQry.query())
+ .parameters(mapQry.parameters(), mapQry.parameterIndexes()));
}
IgniteProductVersion minNodeVer = cctx.shared().exchange().minimumNodeVersion(topVer);
@@ -608,6 +609,12 @@ public class GridReduceQueryExecutor {
if (oldStyle && distributedJoins)
throw new CacheException("Failed to enable distributed joins. Topology contains older data nodes.");
+ // Always enforce join order on map side to have consistent behavior.
+ int flags = GridH2QueryRequest.FLAG_ENFORCE_JOIN_ORDER;
+
+ if (distributedJoins)
+ flags |= GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS;
+
if (send(nodes,
oldStyle ?
new GridQueryRequest(qryReqId,
@@ -626,7 +633,7 @@ public class GridReduceQueryExecutor {
.tables(distributedJoins ? qry.tables() : null)
.partitions(convert(partsMap))
.queries(mapQrys)
- .flags(distributedJoins ? GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS : 0)
+ .flags(flags)
.timeout(timeoutMillis),
oldStyle && partsMap != null ? new ExplicitPartitionsSpecializer(partsMap) : null,
distributedJoins)
@@ -835,14 +842,6 @@ public class GridReduceQueryExecutor {
}
/**
- * @param idx Table index.
- * @return Table name.
- */
- private static String table(int idx) {
- return GridSqlQuerySplitter.table(idx).getSQL();
- }
-
- /**
* Gets or creates new fake table for index.
*
* @param c Connection.
@@ -860,7 +859,7 @@ public class GridReduceQueryExecutor {
try {
if ((tbls = fakeTbls).size() == idx) { // Double check inside of lock.
try (Statement stmt = c.createStatement()) {
- stmt.executeUpdate("CREATE TABLE " + table(idx) +
+ stmt.executeUpdate("CREATE TABLE " + mergeTableIdentifier(idx) +
"(fake BOOL) ENGINE \"" + GridThreadLocalTable.Engine.class.getName() + '"');
}
catch (SQLException e) {
@@ -1117,7 +1116,8 @@ public class GridReduceQueryExecutor {
List<List<?>> lists = new ArrayList<>();
for (int i = 0, mapQrys = qry.mapQueries().size(); i < mapQrys; i++) {
- ResultSet rs = h2.executeSqlQueryWithTimer(space, c, "SELECT PLAN FROM " + table(i), null, false, 0, null);
+ ResultSet rs = h2.executeSqlQueryWithTimer(space, c,
+ "SELECT PLAN FROM " + mergeTableIdentifier(i), null, false, 0, null);
lists.add(F.asList(getPlan(rs)));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
index 884173f..e5dbf33 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
@@ -50,6 +50,11 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
*/
public static int FLAG_DISTRIBUTED_JOINS = 1;
+ /**
+ * Remote map query executor will enforce join order for the received map queries.
+ */
+ public static int FLAG_ENFORCE_JOIN_ORDER = 1 << 1;
+
/** */
private long reqId;
@@ -209,6 +214,8 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
* @return {@code this}.
*/
public GridH2QueryRequest flags(int flags) {
+ assert flags >= 0 && flags <= 255: flags;
+
this.flags = (byte)flags;
return this;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
index d5f02eb..cd38d31 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractFieldsQuerySelfTest.java
@@ -368,7 +368,7 @@ public abstract class IgniteCacheAbstractFieldsQuerySelfTest extends GridCommonA
if (cacheMode() == PARTITIONED) {
assertEquals(2, res.size());
- assertTrue(((String)res.get(1).get(0)).contains(GridSqlQuerySplitter.table(0).getSQL()));
+ assertTrue(((String)res.get(1).get(0)).contains(GridSqlQuerySplitter.mergeTableIdentifier(0)));
}
else
assertEquals(1, res.size());
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 06afe7c..432ed34 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -33,8 +33,8 @@ import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheKeyConfiguration;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
@@ -79,6 +79,11 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
return cfg;
}
+ @Override
+ protected long getTestTimeout() {
+ return 100_000_000;
+ }
+
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
startGridsMultiThreaded(3, false);
@@ -271,9 +276,9 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
String select = "select o.name n1, p.name n2 from Person2 p, Organization o where p.orgId = o._key and o._key=1" +
" union select o.name n1, p.name n2 from Person2 p, Organization o where p.orgId = o._key and o._key=2";
- String plan = (String)c.query(new SqlFieldsQuery("explain " + select)
+ String plan = c.query(new SqlFieldsQuery("explain " + select)
.setDistributedJoins(true).setEnforceJoinOrder(true))
- .getAll().get(0).get(0);
+ .getAll().toString();
X.println("Plan : " + plan);
@@ -285,9 +290,9 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
select = "select * from (" + select + ")";
- plan = (String)c.query(new SqlFieldsQuery("explain " + select)
+ plan = c.query(new SqlFieldsQuery("explain " + select)
.setDistributedJoins(true).setEnforceJoinOrder(true))
- .getAll().get(0).get(0);
+ .getAll().toString();
X.println("Plan : " + plan);
@@ -548,15 +553,15 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
"\"orgRepl\".Organization o",
"where p.affKey = o._key", true);
- checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
- "(select * from \"persPart\".Person2) p",
- "\"orgPart\".Organization o",
- "where p._key = o._key", false);
-
- checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
- "\"persPart\".Person2 p",
- "(select * from \"orgPart\".Organization) o",
- "where p._key = o._key", false);
+ // TODO Now we can not analyze subqueries to decide if we are collocated or not.
+// checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
+// "(select * from \"persPart\".Person2) p",
+// "\"orgPart\".Organization o",
+// "where p._key = o._key", false);
+// checkNoBatchedJoin(persPart, "select p._key k1, o._key k2 ",
+// "\"persPart\".Person2 p",
+// "(select * from \"orgPart\".Organization) o",
+// "where p._key = o._key", false);
// Join multiple.
@@ -576,26 +581,32 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
sql);
sql = "select o.k1, p1._key k2, p2._key k3 from " +
- "(select o1._key k1, o2._key k2 from \"orgRepl\".Organization o1, \"orgRepl2\".Organization o2 where o1._key > o2._key) o, " +
- "\"persPartAff\".Person2 p1, \"persPart\".Person2 p2 where p1._key=p2._key and p2.orgId = o.k1";
+ "(select o1._key k1, o2._key k2 " +
+ "from \"orgRepl\".Organization o1, \"orgRepl2\".Organization o2 " +
+ "where o1._key > o2._key) o, " +
+ "\"persPartAff\".Person2 p1, \"persPart\".Person2 p2 " +
+ "where p1._key=p2._key and p2.orgId = o.k1";
checkQueryPlan(persPart,
false,
- 1,
+ 0,
sql,
- "persPartAff", "persPart", "batched:unicast", "orgRepl");
+ "persPartAff", "persPart", "orgRepl");
checkQueryFails(persPart, sql, true);
- sql = "select o.k1, p._key k2 from " +
- "(select o1._key k1, p1._key k2 from \"orgRepl\".Organization o1, \"persPart\".Person2 p1 where o1._key = p1.orgId) o, " +
- "\"persPartAff\".Person2 p where p._key=o.k1";
+ sql = "select o.ok, p._key from " +
+ "(select o1._key ok, p1._key pk " +
+ "from \"orgRepl\".Organization o1, \"persPart\".Person2 p1 " +
+ "where o1._key = p1.orgId) o, " +
+ "\"persPartAff\".Person2 p where p._key=o.ok";
checkQueryPlan(persPart,
false,
1,
sql,
- "FROM \"persPart\"", "INNER JOIN \"orgRepl\"", "INNER JOIN \"persPartAff\"", "batched:broadcast");
+ "FROM \"persPart\"", "INNER JOIN \"orgRepl\"",
+ "INNER JOIN \"persPartAff\"", "batched:unicast");
checkQueryFails(persPart, sql, true);
}
@@ -657,7 +668,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
{
String sql = "select p1._key k1, p2._key k2, o._key k3 " +
"from \"persPartAff\".Person2 p1, \"persPart\".Person2 p2, \"orgRepl\".Organization o " +
- "where p1._key=p2._key and p2.orgId = o._key";
+ "where p1._key=p2.name and p2.orgId = o._key";
checkQueryPlan(persPart,
false,
@@ -666,21 +677,39 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
"batched:unicast");
sql = "select p1._key k1, p2._key k2, o._key k3 " +
- "from \"orgRepl\".Organization o, \"persPartAff\".Person2 p1, \"persPart\".Person2 p2 " +
+ "from \"persPartAff\".Person2 p1, \"persPart\".Person2 p2, \"orgRepl\".Organization o " +
"where p1._key=p2._key and p2.orgId = o._key";
checkQueryPlan(persPart,
false,
+ 0,
+ sql);
+
+ sql = "select p1._key k1, p2._key k2, o._key k3 " +
+ "from \"orgRepl\".Organization o, \"persPartAff\".Person2 p1, \"persPart\".Person2 p2 " +
+ "where p1._key=p2.name and p2.orgId = o._key";
+
+ checkQueryPlan(persPart,
+ false,
1,
sql,
"batched:unicast");
sql = "select p1._key k1, p2._key k2, o._key k3 " +
- "from \"persPartAff\".Person2 p1, \"orgRepl\".Organization o, \"persPart\".Person2 p2 " +
+ "from \"orgRepl\".Organization o, \"persPartAff\".Person2 p1, \"persPart\".Person2 p2 " +
"where p1._key=p2._key and p2.orgId = o._key";
checkQueryPlan(persPart,
false,
+ 0,
+ sql);
+
+ sql = "select p1._key k1, p2._key k2, o._key k3 " +
+ "from (select * from \"orgRepl\".Organization) o, \"persPartAff\".Person2 p1, \"persPart\".Person2 p2 " +
+ "where p1._key=p2.name and p2.orgId = o._key";
+
+ checkQueryPlan(persPart,
+ false,
1,
sql,
"batched:unicast");
@@ -691,9 +720,8 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
checkQueryPlan(persPart,
false,
- 1,
- sql,
- "batched:unicast");
+ 0,
+ sql);
}
}
finally {
@@ -823,17 +851,28 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
boolean enforceJoinOrder,
int expBatchedJoins,
String sql,
- String...expText) {
+ String...expText
+ ) {
+ checkQueryPlan(cache,
+ enforceJoinOrder,
+ expBatchedJoins,
+ new SqlFieldsQuery(sql),
+ expText);
+
+ sql = "select * from (" + sql + ")";
+
checkQueryPlan(cache,
enforceJoinOrder,
expBatchedJoins,
new SqlFieldsQuery(sql),
expText);
+ sql = "select * from (" + sql + ")";
+
checkQueryPlan(cache,
enforceJoinOrder,
expBatchedJoins,
- new SqlFieldsQuery("select * from (" + sql + ")"),
+ new SqlFieldsQuery(sql),
expText);
}
@@ -854,7 +893,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
String plan = queryPlan(cache, qry);
- log.info("Plan: " + plan);
+ log.info("\n Plan:\n" + plan);
assertEquals("Unexpected number of batched joins in plan [plan=" + plan + ", qry=" + qry + ']',
expBatchedJoins,
http://git-wip-us.apache.org/repos/asf/ignite/blob/3737407b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index b909b36..d3ff902 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -53,6 +53,12 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
/** */
+ private static final String TEST_SCHEMA = "SCH";
+
+ /** */
+ private static final String TEST_CACHE = "my-cache";
+
+ /** */
private static Ignite ignite;
/** {@inheritDoc} */
@@ -69,12 +75,14 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
// Cache.
CacheConfiguration cc = defaultCacheConfiguration();
+ cc.setName(TEST_CACHE);
cc.setCacheMode(CacheMode.PARTITIONED);
cc.setAtomicityMode(CacheAtomicityMode.ATOMIC);
cc.setNearConfiguration(null);
cc.setWriteSynchronizationMode(FULL_SYNC);
cc.setRebalanceMode(SYNC);
cc.setSwapEnabled(false);
+ cc.setSqlSchema(TEST_SCHEMA);
cc.setSqlFunctionClasses(GridQueryParsingTest.class);
cc.setIndexedTypes(
String.class, Address.class,
@@ -163,7 +171,7 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
checkQuery("select avg(old) from Person, Address where Person.addrId = Address.id " +
"and lower(Address.street) = lower(?)");
- checkQuery("select name, date from Person");
+ checkQuery("select name, name, date, date d from Person");
checkQuery("select distinct name, date from Person");
checkQuery("select * from Person p");
checkQuery("select * from Person");
@@ -240,16 +248,16 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
checkQuery("select street from Person p, (select a.street from Address a where a.street is not null) ");
checkQuery("select addr.street from Person p, (select a.street from Address a where a.street is not null) addr");
- checkQuery("select p.name n from \"\".Person p order by p.old + 10");
+ checkQuery("select p.name n from sch.Person p order by p.old + 10");
- checkQuery("select case when p.name is null then 'Vasya' end x from \"\".Person p");
- checkQuery("select case when p.name like 'V%' then 'Vasya' else 'Other' end x from \"\".Person p");
- checkQuery("select case when upper(p.name) = 'VASYA' then 'Vasya' when p.name is not null then p.name else 'Other' end x from \"\".Person p");
+ checkQuery("select case when p.name is null then 'Vasya' end x from sch.Person p");
+ checkQuery("select case when p.name like 'V%' then 'Vasya' else 'Other' end x from sch.Person p");
+ checkQuery("select case when upper(p.name) = 'VASYA' then 'Vasya' when p.name is not null then p.name else 'Other' end x from sch.Person p");
- checkQuery("select case p.name when 'Vasya' then 1 end z from \"\".Person p");
- checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 end z from \"\".Person p");
- checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 else 3 end z from \"\".Person p");
- checkQuery("select case p.name when 'Vasya' then 1 else 3 end z from \"\".Person p");
+ checkQuery("select case p.name when 'Vasya' then 1 end z from sch.Person p");
+ checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 end z from sch.Person p");
+ checkQuery("select case p.name when 'Vasya' then 1 when 'Petya' then 2 else 3 end z from sch.Person p");
+ checkQuery("select case p.name when 'Vasya' then 1 else 3 end z from sch.Person p");
checkQuery("select count(*) as a from Person union select count(*) as a from Address");
checkQuery("select old, count(*) as a from Person group by old union select 1, count(*) as a from Address");
@@ -262,6 +270,17 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
checkQuery("(select name from Person limit 4) UNION (select street from Address limit 1) limit ? offset ?");
checkQuery("(select 2 a) union all (select 1) order by 1");
checkQuery("(select 2 a) union all (select 1) order by a desc nulls first limit ? offset ?");
+
+ checkQuery("select public.\"#\".\"@\" from (select 1 as \"@\") \"#\"");
+// checkQuery("select sch.\"#\".\"@\" from (select 1 as \"@\") \"#\""); // Illegal query.
+ checkQuery("select \"#\".\"@\" from (select 1 as \"@\") \"#\"");
+ checkQuery("select \"@\" from (select 1 as \"@\") \"#\"");
+ checkQuery("select sch.\"#\".old from sch.Person \"#\"");
+ checkQuery("select sch.\"#\".old from Person \"#\"");
+ checkQuery("select \"#\".old from Person \"#\"");
+ checkQuery("select old from Person \"#\"");
+// checkQuery("select Person.old from Person \"#\""); // Illegal query.
+ checkQuery("select sch.\"#\".* from Person \"#\"");
}
/** */
@@ -374,7 +393,7 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
IgniteH2Indexing idx = U.field(qryProcessor, "idx");
- return (JdbcConnection)idx.connectionForSpace(null);
+ return (JdbcConnection)idx.connectionForSpace(TEST_CACHE);
}
/**
@@ -416,7 +435,7 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
private void checkQuery(String qry) throws Exception {
Prepared prepared = parse(qry);
- GridSqlStatement gQry = new GridSqlQueryParser().parse(prepared);
+ GridSqlStatement gQry = new GridSqlQueryParser(false).parse(prepared);
String res = gQry.getSQL();
[46/50] [abbrv] ignite git commit: IGNITE-4754 SharedRDDExample
missed in example JAR file
Posted by av...@apache.org.
IGNITE-4754 SharedRDDExample missed in example JAR file
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a779c5f9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a779c5f9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a779c5f9
Branch: refs/heads/master
Commit: a779c5f9439bd797d9c662f32663780c09271672
Parents: 4dd3d23
Author: Evgenii Zhuravlev <ez...@gridgain.com>
Authored: Mon Feb 27 11:45:48 2017 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Mon Feb 27 14:11:57 2017 +0300
----------------------------------------------------------------------
examples/pom-standalone-lgpl.xml | 6 ++++++
examples/pom-standalone.xml | 6 ++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a779c5f9/examples/pom-standalone-lgpl.xml
----------------------------------------------------------------------
diff --git a/examples/pom-standalone-lgpl.xml b/examples/pom-standalone-lgpl.xml
index c09cc6b..f664514 100644
--- a/examples/pom-standalone-lgpl.xml
+++ b/examples/pom-standalone-lgpl.xml
@@ -32,6 +32,7 @@
<!-- will be changed by profile activation. allows to combine profiles. -->
<lgpl.folder>src/main/java</lgpl.folder>
<java8.folder>src/main/java</java8.folder>
+ <spark.folder>src/main/java</spark.folder>
<java.ver>1.7</java.ver>
</properties>
@@ -100,6 +101,10 @@
<profile>
<id>scala</id>
+ <properties>
+ <spark.folder>src/main/spark</spark.folder>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ignite</groupId>
@@ -219,6 +224,7 @@
<source>schema-import/src/main/java</source>
<source>${lgpl.folder}</source>
<source>${java8.folder}</source>
+ <source>${spark.folder}</source>
</sources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a779c5f9/examples/pom-standalone.xml
----------------------------------------------------------------------
diff --git a/examples/pom-standalone.xml b/examples/pom-standalone.xml
index b3843e6..029ca77 100644
--- a/examples/pom-standalone.xml
+++ b/examples/pom-standalone.xml
@@ -32,6 +32,7 @@
<!-- will be changed by profile activation. allows to combine profiles. -->
<lgpl.folder>src/main/java</lgpl.folder>
<java8.folder>src/main/java</java8.folder>
+ <spark.folder>src/main/java</spark.folder>
<java.ver>1.7</java.ver>
</properties>
@@ -100,6 +101,10 @@
<profile>
<id>scala</id>
+ <properties>
+ <spark.folder>src/main/spark</spark.folder>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.ignite</groupId>
@@ -220,6 +225,7 @@
<source>schema-import/src/main/java</source>
<source>${lgpl.folder}</source>
<source>${java8.folder}</source>
+ <source>${spark.folder}</source>
</sources>
</configuration>
</execution>
[25/50] [abbrv] ignite git commit: Removed redundant call
Posted by av...@apache.org.
Removed redundant call
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f0ef8420
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f0ef8420
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f0ef8420
Branch: refs/heads/master
Commit: f0ef842097b61dad364c98e79e125b6a9b1ffc31
Parents: 990f64d
Author: Alexander Paschenko <al...@gmail.com>
Authored: Tue Feb 21 15:59:31 2017 +0300
Committer: Alexander Paschenko <al...@gmail.com>
Committed: Tue Feb 21 15:59:31 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/processors/query/h2/sql/DmlAstUtils.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f0ef8420/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
index f96e5e3..974a5b8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
@@ -109,8 +109,6 @@ public final class DmlAstUtils {
args[i].addChild(row[i]);
}
- sel.getSQL();
-
return sel;
}
else {
[32/50] [abbrv] ignite git commit: ignite-3860 - parsing fixed
Posted by av...@apache.org.
ignite-3860 - parsing fixed
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/efb86680
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/efb86680
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/efb86680
Branch: refs/heads/master
Commit: efb866803e9831c86b1c4b1bab93cf12076c8a73
Parents: bf337bc
Author: Sergi Vladykin <se...@gmail.com>
Authored: Wed Feb 22 02:31:54 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Wed Feb 22 02:31:54 2017 +0300
----------------------------------------------------------------------
.../query/h2/sql/GridSqlQueryParser.java | 20 +++++++++-----
.../query/h2/sql/GridQueryParsingTest.java | 28 ++++++++++++++++++++
2 files changed, 41 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/efb86680/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index f5ad397..024529c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -344,7 +344,7 @@ public class GridSqlQueryParser {
/**
* @param filter Filter.
*/
- private GridSqlElement parseTable(TableFilter filter) {
+ private GridSqlElement parseTableFilter(TableFilter filter) {
GridSqlElement res = (GridSqlElement)h2ObjToGridObj.get(filter);
if (res == null) {
@@ -369,9 +369,15 @@ public class GridSqlQueryParser {
GridSqlElement res = (GridSqlElement)h2ObjToGridObj.get(tbl);
if (res == null) {
+ // We can't cache simple tables because otherwise it will be the same instance for all
+ // table filters. Thus we will not be able to distinguish one table filter from another.
+ // Table here is semantically equivalent to a table filter.
if (tbl instanceof TableBase)
- res = new GridSqlTable(tbl);
- else if (tbl instanceof TableView) {
+ return new GridSqlTable(tbl);
+
+ // Other stuff can be cached because we will have separate instances in
+ // different table filters anyways. Thus the semantics will be correct.
+ if (tbl instanceof TableView) {
Query qry = VIEW_QUERY.get((TableView) tbl);
res = new GridSqlSubquery(parseQuery(qry));
@@ -438,7 +444,7 @@ public class GridSqlQueryParser {
for (int i = 0; i < tableFilters.size(); i++) {
TableFilter f = tableFilters.get(i);
- GridSqlElement gridFilter = parseTable(f);
+ GridSqlElement gridFilter = parseTableFilter(f);
from = from == null ? gridFilter : new GridSqlJoin(from, gridFilter, f.isJoinOuter(),
parseExpression(f.getJoinCondition(), false));
@@ -606,7 +612,7 @@ public class GridSqlQueryParser {
res = new GridSqlDelete();
h2ObjToGridObj.put(del, res);
- GridSqlElement tbl = parseTable(DELETE_FROM.get(del));
+ GridSqlElement tbl = parseTableFilter(DELETE_FROM.get(del));
GridSqlElement where = parseExpression(DELETE_WHERE.get(del), true);
GridSqlElement limit = parseExpression(DELETE_LIMIT.get(del), true);
res.from(tbl).where(where).limit(limit);
@@ -626,7 +632,7 @@ public class GridSqlQueryParser {
res = new GridSqlUpdate();
h2ObjToGridObj.put(update, res);
- GridSqlElement tbl = parseTable(UPDATE_TARGET.get(update));
+ GridSqlElement tbl = parseTableFilter(UPDATE_TARGET.get(update));
List<Column> srcCols = UPDATE_COLUMNS.get(update);
Map<Column, Expression> srcSet = UPDATE_SET.get(update);
@@ -826,7 +832,7 @@ public class GridSqlQueryParser {
ExpressionColumn expCol = (ExpressionColumn)expression;
return new GridSqlColumn(expCol.getColumn(),
- parseTable(expCol.getTableFilter()),
+ parseTableFilter(expCol.getTableFilter()),
SCHEMA_NAME.get(expCol),
expCol.getOriginalTableAliasName(),
expCol.getColumnName());
http://git-wip-us.apache.org/repos/asf/ignite/blob/efb86680/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
index d3ff902..b869da9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/GridQueryParsingTest.java
@@ -283,6 +283,34 @@ public class GridQueryParsingTest extends GridCommonAbstractTest {
checkQuery("select sch.\"#\".* from Person \"#\"");
}
+ /**
+ * Query AST transformation heavily depends on this behavior.
+ *
+ * @throws Exception If failed.
+ */
+ public void testParseTableFilter() throws Exception {
+ Prepared prepared = parse("select Person.old, p1.old from Person, Person p1");
+
+ GridSqlSelect select = (GridSqlSelect)new GridSqlQueryParser(false).parse(prepared);
+
+ GridSqlJoin join = (GridSqlJoin)select.from();
+
+ GridSqlTable tbl1 = (GridSqlTable)join.leftTable();
+ GridSqlAlias tbl2Alias = (GridSqlAlias)join.rightTable();
+ GridSqlTable tbl2 = tbl2Alias.child();
+
+ // Must be distinct objects, even if it is the same table.
+ assertNotSame(tbl1, tbl2);
+
+ GridSqlColumn col1 = (GridSqlColumn)select.column(0);
+ GridSqlColumn col2 = (GridSqlColumn)select.column(1);
+
+ assertSame(tbl1, col1.expressionInFrom());
+
+ // Alias in FROM must be included in column.
+ assertSame(tbl2Alias, col2.expressionInFrom());
+ }
+
/** */
public void testParseMerge() throws Exception {
/* Plain rows w/functions, operators, defaults, and placeholders. */
[15/50] [abbrv] ignite git commit: Implemented.
Posted by av...@apache.org.
Implemented.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/64ba13b0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/64ba13b0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/64ba13b0
Branch: refs/heads/master
Commit: 64ba13b0a3be6acbf7d629029b460a39c2e2b388
Parents: 4eac51c
Author: AMRepo <an...@gmail.com>
Authored: Mon Feb 20 21:24:29 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 11:52:40 2017 +0300
----------------------------------------------------------------------
.../configuration/CacheConfiguration.java | 48 ++++
.../processors/cache/GridCacheProcessor.java | 3 +
.../processors/cache/IgniteCacheProxy.java | 6 +-
.../closure/GridClosureProcessor.java | 2 +-
.../processors/query/GridQueryIndexing.java | 27 +-
.../processors/query/GridQueryProcessor.java | 141 +++-------
.../messages/GridQueryNextPageRequest.java | 29 +-
.../messages/GridQueryNextPageResponse.java | 29 +-
.../cache/query/GridCacheTwoStepQuery.java | 17 ++
.../processors/query/h2/IgniteH2Indexing.java | 235 ++++++++++++++--
.../query/h2/opt/DistributedJoinMode.java | 51 ++++
.../query/h2/opt/GridH2IndexBase.java | 264 +++++++++++++-----
.../query/h2/opt/GridH2QueryContext.java | 84 ++++--
.../query/h2/opt/GridH2TreeIndex.java | 232 ++++++++++++----
.../query/h2/twostep/GridMapQueryExecutor.java | 227 +++++++++++----
.../query/h2/twostep/GridMergeIndex.java | 39 ++-
.../h2/twostep/GridReduceQueryExecutor.java | 69 +++--
.../h2/twostep/msg/GridH2IndexRangeRequest.java | 60 +++-
.../twostep/msg/GridH2IndexRangeResponse.java | 62 ++++-
.../h2/twostep/msg/GridH2QueryRequest.java | 5 +
.../query/IgniteSqlSegmentedIndexSelfTest.java | 263 ++++++++++++++++++
.../query/IgniteSqlSplitterSelfTest.java | 139 +++++++++-
.../h2/GridIndexingSpiAbstractSelfTest.java | 26 +-
.../FetchingQueryCursorStressTest.java | 277 +++++++++++++++++++
.../IgniteCacheQuerySelfTestSuite.java | 2 +
25 files changed, 1917 insertions(+), 420 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
index 0656dda..149f25a 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CacheConfiguration.java
@@ -223,6 +223,9 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
/** Default threshold for concurrent loading of keys from {@link CacheStore}. */
public static final int DFLT_CONCURRENT_LOAD_ALL_THRESHOLD = 5;
+ /** Default SQL query parallelism level */
+ public static final int DFLT_SQL_QUERY_PARALLELISM_LVL = 1;
+
/** Cache name. */
private String name;
@@ -410,6 +413,9 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
/** Query entities. */
private Collection<QueryEntity> qryEntities;
+ /** */
+ private int qryParallelism = DFLT_SQL_QUERY_PARALLELISM_LVL;
+
/** Empty constructor (all values are initialized to their defaults). */
public CacheConfiguration() {
/* No-op. */
@@ -462,6 +468,7 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
interceptor = cc.getInterceptor();
invalidate = cc.isInvalidate();
isReadThrough = cc.isReadThrough();
+ qryParallelism = cc.getQueryParallelism();
isWriteThrough = cc.isWriteThrough();
storeKeepBinary = cc.isStoreKeepBinary() != null ? cc.isStoreKeepBinary() : DFLT_STORE_KEEP_BINARY;
listenerConfigurations = cc.listenerConfigurations;
@@ -2108,6 +2115,47 @@ public class CacheConfiguration<K, V> extends MutableConfiguration<K, V> {
}
/**
+ * Defines a hint to query execution engine on desired degree of parallelism within a single node.
+ * Query executor may or may not use this hint depending on estimated query costs. Query executor may define
+ * certain restrictions on parallelism depending on query type and/or cache type.
+ * <p>
+ * As of {@code Apache Ignite 1.9} this hint is only supported for SQL queries with the following restrictions:
+ * <ul>
+ * <li>Hint cannot be used for {@code REPLICATED} cache, exception is thrown otherwise</li>
+ * <li>All caches participating in query must have the same degree of parallelism, exception is thrown
+ * otherwise</li>
+ * </ul>
+ * These restrictions will be removed in future versions of Apache Ignite.
+ * <p>
+ * Defaults to {@code 1}.
+ */
+ public int getQueryParallelism() {
+ return qryParallelism;
+ }
+
+ /**
+ * Defines a hint to query execution engine on desired degree of parallelism within a single node.
+ * Query executor may or may not use this hint depending on estimated query costs. Query executor may define
+ * certain restrictions on parallelism depending on query type and/or cache type.
+ * <p>
+ * As of {@code Apache Ignite 1.9} this hint is only supported for SQL queries with the following restrictions:
+ * <ul>
+ * <li>Hint cannot be used for {@code REPLICATED} cache, exception is thrown otherwise</li>
+ * <li>All caches participating in query must have the same degree of parallelism, exception is thrown
+ * otherwise</li>
+ * </ul>
+ * These restrictions will be removed in future versions of Apache Ignite.
+ *
+ * @param qryParallelism Query parallelizm level.
+ * @return {@code this} for chaining.
+ */
+ public CacheConfiguration<K,V> setQueryParallelism(int qryParallelism) {
+ this.qryParallelism = qryParallelism;
+
+ return this;
+ }
+
+ /**
* Gets topology validator.
* <p>
* See {@link TopologyValidator} for details.
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 7093403..c3e3f3b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -269,6 +269,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
if (cfg.getCacheMode() == REPLICATED)
cfg.setBackups(Integer.MAX_VALUE);
+ if( cfg.getQueryParallelism() > 1 && cfg.getCacheMode() != PARTITIONED)
+ throw new IgniteCheckedException("Cache index segmentation is supported for PARTITIONED mode only.");
+
if (cfg.getAffinityMapper() == null)
cfg.setAffinityMapper(cacheObjCtx.defaultAffMapper());
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
index 1381670..f806d05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxy.java
@@ -729,12 +729,8 @@ public class IgniteCacheProxy<K, V> extends AsyncSupportAdapter<IgniteCache<K, V
final SqlQuery p = (SqlQuery)qry;
if (isReplicatedDataNode() || ctx.isLocal() || qry.isLocal())
- return (QueryCursor<R>)new QueryCursorImpl<>(new Iterable<Cache.Entry<K, V>>() {
- @Override public Iterator<Cache.Entry<K, V>> iterator() {
- return ctx.kernalContext().query().queryLocal(ctx, p,
+ return (QueryCursor<R>)ctx.kernalContext().query().queryLocal(ctx, p,
opCtxCall != null && opCtxCall.isKeepBinary());
- }
- });
return (QueryCursor<R>)ctx.kernalContext().query().queryTwoStep(ctx, p);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
index 20fb6a0..61ed8a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/closure/GridClosureProcessor.java
@@ -902,7 +902,7 @@ public class GridClosureProcessor extends GridProcessorAdapter {
* @return Future.
* @throws IgniteCheckedException Thrown in case of any errors.
*/
- private <R> IgniteInternalFuture<R> callLocal(@Nullable final Callable<R> c, byte plc)
+ public <R> IgniteInternalFuture<R> callLocal(@Nullable final Callable<R> c, byte plc)
throws IgniteCheckedException {
if (c == null)
return new GridFinishedFuture<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index ca04724..37f0ade 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -84,35 +84,26 @@ public interface GridQueryIndexing {
/**
* Queries individual fields (generally used by JDBC drivers).
*
- * @param spaceName Space name.
+ * @param cctx Cache context.
* @param qry Query.
- * @param params Query parameters.
* @param filter Space name and key filter.
- * @param enforceJoinOrder Enforce join order of tables in the query.
- * @param timeout Query timeout in milliseconds.
* @param cancel Query cancel.
- * @return Query result.
- * @throws IgniteCheckedException If failed.
+ * @return Cursor.
*/
- public GridQueryFieldsResult queryLocalSqlFields(@Nullable String spaceName, String qry,
- Collection<Object> params, IndexingQueryFilter filter, boolean enforceJoinOrder, int timeout,
- GridQueryCancel cancel) throws IgniteCheckedException;
+ public <K, V> QueryCursor<List<?>> queryLocalSqlFields(GridCacheContext<?, ?> cctx, SqlFieldsQuery qry,
+ IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException;
/**
* Executes regular query.
*
- * @param spaceName Space name.
+ * @param cctx Cache context.
* @param qry Query.
- * @param alias Table alias used in Query.
- * @param params Query parameters.
- * @param type Query return type.
* @param filter Space name and key filter.
- * @return Queried rows.
- * @throws IgniteCheckedException If failed.
+ * @param keepBinary Keep binary flag.
+ * @return Cursor.
*/
- public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName, String qry,
- String alias, Collection<Object> params, GridQueryTypeDescriptor type, IndexingQueryFilter filter)
- throws IgniteCheckedException;
+ public <K, V> QueryCursor<Cache.Entry<K,V>> queryLocalSql(GridCacheContext<?, ?> cctx, SqlQuery qry,
+ IndexingQueryFilter filter, boolean keepBinary) throws IgniteCheckedException;
/**
* Executes text query.
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index ee9224b..85744d5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -754,42 +754,6 @@ public class GridQueryProcessor extends GridProcessorAdapter {
INDEXING.module() + " to classpath or moving it from 'optional' to 'libs' folder).");
}
- /**
- * @param space Space.
- * @param clause Clause.
- * @param params Parameters collection.
- * @param resType Result type.
- * @param filters Filters.
- * @return Key/value rows.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> query(final String space, final String clause,
- final Collection<Object> params, final String resType, final IndexingQueryFilter filters)
- throws IgniteCheckedException {
- checkEnabled();
-
- if (!busyLock.enterBusy())
- throw new IllegalStateException("Failed to execute query (grid is stopping).");
-
- try {
- final GridCacheContext<?, ?> cctx = ctx.cache().internalCache(space).context();
-
- return executeQuery(GridCacheQueryType.SQL_FIELDS, clause, cctx, new IgniteOutClosureX<GridCloseableIterator<IgniteBiTuple<K, V>>>() {
- @Override public GridCloseableIterator<IgniteBiTuple<K, V>> applyx() throws IgniteCheckedException {
- TypeDescriptor type = typesByName.get(new TypeName(space, resType));
-
- if (type == null || !type.registered())
- throw new CacheException("Failed to find SQL table for type: " + resType);
-
- return idx.queryLocalSql(space, clause, null, params, type, filters);
- }
- }, false);
- }
- finally {
- busyLock.leaveBusy();
- }
- }
/**
* @param cctx Cache context.
@@ -829,11 +793,12 @@ public class GridQueryProcessor extends GridProcessorAdapter {
throw new IllegalStateException("Failed to execute query (grid is stopping).");
try {
- return executeQuery(GridCacheQueryType.SQL, qry.getSql(), cctx, new IgniteOutClosureX<QueryCursor<Cache.Entry<K, V>>>() {
- @Override public QueryCursor<Cache.Entry<K, V>> applyx() throws IgniteCheckedException {
- return idx.queryTwoStep(cctx, qry);
- }
- }, true);
+ return executeQuery(GridCacheQueryType.SQL, qry.getSql(), cctx,
+ new IgniteOutClosureX<QueryCursor<Cache.Entry<K, V>>>() {
+ @Override public QueryCursor<Cache.Entry<K, V>> applyx() throws IgniteCheckedException {
+ return idx.queryTwoStep(cctx, qry);
+ }
+ }, true);
}
catch (IgniteCheckedException e) {
throw new IgniteException(e);
@@ -849,7 +814,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* @param keepBinary Keep binary flag.
* @return Cursor.
*/
- public <K, V> Iterator<Cache.Entry<K, V>> queryLocal(
+ public <K, V> QueryCursor<Cache.Entry<K, V>> queryLocal(
final GridCacheContext<?, ?> cctx,
final SqlQuery qry,
final boolean keepBinary
@@ -859,54 +824,25 @@ public class GridQueryProcessor extends GridProcessorAdapter {
try {
return executeQuery(GridCacheQueryType.SQL, qry.getSql(), cctx,
- new IgniteOutClosureX<Iterator<Cache.Entry<K, V>>>() {
- @Override public Iterator<Cache.Entry<K, V>> applyx() throws IgniteCheckedException {
- String space = cctx.name();
+ new IgniteOutClosureX<QueryCursor<Cache.Entry<K, V>>>() {
+ @Override public QueryCursor<Cache.Entry<K, V>> applyx() throws IgniteCheckedException {
String type = qry.getType();
- String sqlQry = qry.getSql();
- Object[] params = qry.getArgs();
- TypeDescriptor typeDesc = typesByName.get(
- new TypeName(
- space,
+ GridQueryProcessor.TypeDescriptor typeDesc = typesByName.get(
+ new GridQueryProcessor.TypeName(
+ cctx.name(),
type));
if (typeDesc == null || !typeDesc.registered())
throw new CacheException("Failed to find SQL table for type: " + type);
- final GridCloseableIterator<IgniteBiTuple<K, V>> i = idx.queryLocalSql(
- space,
- qry.getSql(),
- qry.getAlias(),
- F.asList(params),
- typeDesc,
- idx.backupFilter(requestTopVer.get(), null));
+ qry.setType(typeDesc.name());
sendQueryExecutedEvent(
- sqlQry,
- params);
-
- return new ClIter<Cache.Entry<K, V>>() {
- @Override public void close() throws Exception {
- i.close();
- }
-
- @Override public boolean hasNext() {
- return i.hasNext();
- }
-
- @Override public Cache.Entry<K, V> next() {
- IgniteBiTuple<K, V> t = i.next();
-
- return new CacheEntryImpl<>(
- (K)cctx.unwrapBinaryIfNeeded(t.getKey(), keepBinary, false),
- (V)cctx.unwrapBinaryIfNeeded(t.getValue(), keepBinary, false));
- }
-
- @Override public void remove() {
- throw new UnsupportedOperationException();
- }
- };
+ qry.getSql(),
+ qry.getArgs());
+
+ return idx.queryLocalSql(cctx, qry, idx.backupFilter(requestTopVer.get(), null), keepBinary);
}
}, true);
}
@@ -994,13 +930,6 @@ public class GridQueryProcessor extends GridProcessorAdapter {
}
/**
- * Closeable iterator.
- */
- private interface ClIter<X> extends AutoCloseable, Iterator<X> {
- // No-op.
- }
-
- /**
* @param cctx Cache context.
* @param qry Query.
* @return Iterator.
@@ -1010,34 +939,26 @@ public class GridQueryProcessor extends GridProcessorAdapter {
throw new IllegalStateException("Failed to execute query (grid is stopping).");
try {
- final boolean keepBinary = cctx.keepBinary();
-
return executeQuery(GridCacheQueryType.SQL_FIELDS, qry.getSql(), cctx, new IgniteOutClosureX<QueryCursor<List<?>>>() {
@Override public QueryCursor<List<?>> applyx() throws IgniteCheckedException {
- final String space = cctx.name();
- final String sql = qry.getSql();
- final Object[] args = qry.getArgs();
- final GridQueryCancel cancel = new GridQueryCancel();
+ GridQueryCancel cancel = new GridQueryCancel();
- final GridQueryFieldsResult res = idx.queryLocalSqlFields(space, sql, F.asList(args),
- idx.backupFilter(requestTopVer.get(), null), qry.isEnforceJoinOrder(), qry.getTimeout(), cancel);
+ final QueryCursor<List<?>> cursor = idx.queryLocalSqlFields(cctx, qry,
+ idx.backupFilter(requestTopVer.get(), null), cancel);
- QueryCursorImpl<List<?>> cursor = new QueryCursorImpl<>(new Iterable<List<?>>() {
+ return new QueryCursorImpl<List<?>>(new Iterable<List<?>>() {
@Override public Iterator<List<?>> iterator() {
- try {
- sendQueryExecutedEvent(sql, args);
-
- return new GridQueryCacheObjectsIterator(res.iterator(), cctx, keepBinary);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
- }, cancel);
-
- cursor.fieldsMeta(res.metaData());
+ sendQueryExecutedEvent(qry.getSql(), qry.getArgs());
- return cursor;
+ return cursor.iterator();
+ }
+ }, cancel) {
+ @Override public List<GridQueryFieldMetadata> fieldsMeta() {
+ if (cursor instanceof QueryCursorImpl)
+ return ((QueryCursorImpl)cursor).fieldsMeta();
+ return super.fieldsMeta();
+ }
+ };
}
}, true);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageRequest.java
index 1feff5a..acea084 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageRequest.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.processors.query.h2.twostep.messages;
-
import java.nio.ByteBuffer;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.plugin.extensions.communication.Message;
@@ -35,6 +34,9 @@ public class GridQueryNextPageRequest implements Message {
private long qryReqId;
/** */
+ private int segmentId;
+
+ /** */
private int qry;
/** */
@@ -50,11 +52,13 @@ public class GridQueryNextPageRequest implements Message {
/**
* @param qryReqId Query request ID.
* @param qry Query.
+ * @param segmentId Index segment ID.
* @param pageSize Page size.
*/
- public GridQueryNextPageRequest(long qryReqId, int qry, int pageSize) {
+ public GridQueryNextPageRequest(long qryReqId, int qry, int segmentId, int pageSize) {
this.qryReqId = qryReqId;
this.qry = qry;
+ this.segmentId = segmentId;
this.pageSize = pageSize;
}
@@ -72,6 +76,11 @@ public class GridQueryNextPageRequest implements Message {
return qry;
}
+ /** @return Index segment ID */
+ public int segmentId() {
+ return segmentId;
+ }
+
/**
* @return Page size.
*/
@@ -119,6 +128,12 @@ public class GridQueryNextPageRequest implements Message {
writer.incrementState();
+ case 3:
+ if (!writer.writeInt("segmentId", segmentId))
+ return false;
+
+ writer.incrementState();
+
}
return true;
@@ -156,6 +171,14 @@ public class GridQueryNextPageRequest implements Message {
reader.incrementState();
+ case 3:
+ segmentId = reader.readInt("segmentId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
}
return reader.afterMessageRead(GridQueryNextPageRequest.class);
@@ -168,6 +191,6 @@ public class GridQueryNextPageRequest implements Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 3;
+ return 4;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
index 4889069..e85c00b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
@@ -42,6 +42,9 @@ public class GridQueryNextPageResponse implements Message {
private long qryReqId;
/** */
+ private int segmentId;
+
+ /** */
private int qry;
/** */
@@ -73,6 +76,7 @@ public class GridQueryNextPageResponse implements Message {
/**
* @param qryReqId Query request ID.
+ * @param segmentId Index segment ID.
* @param qry Query.
* @param page Page.
* @param allRows All rows count.
@@ -80,12 +84,13 @@ public class GridQueryNextPageResponse implements Message {
* @param vals Values for rows in this page added sequentially.
* @param plainRows Not marshalled rows for local node.
*/
- public GridQueryNextPageResponse(long qryReqId, int qry, int page, int allRows, int cols,
+ public GridQueryNextPageResponse(long qryReqId, int segmentId, int qry, int page, int allRows, int cols,
Collection<Message> vals, Collection<?> plainRows) {
assert vals != null ^ plainRows != null;
assert cols > 0 : cols;
this.qryReqId = qryReqId;
+ this.segmentId = segmentId;
this.qry = qry;
this.page = page;
this.allRows = allRows;
@@ -102,6 +107,13 @@ public class GridQueryNextPageResponse implements Message {
}
/**
+ * @return Index segment ID.
+ */
+ public int segmentId() {
+ return segmentId;
+ }
+
+ /**
* @return Query.
*/
public int query() {
@@ -202,6 +214,12 @@ public class GridQueryNextPageResponse implements Message {
writer.incrementState();
+ case 7:
+ if (!writer.writeInt("segmentId", segmentId))
+ return false;
+
+ writer.incrementState();
+
}
return true;
@@ -271,6 +289,13 @@ public class GridQueryNextPageResponse implements Message {
reader.incrementState();
+ case 7:
+ segmentId = reader.readInt("segmentId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
}
return reader.afterMessageRead(GridQueryNextPageResponse.class);
@@ -283,7 +308,7 @@ public class GridQueryNextPageResponse implements Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 7;
+ return 8;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
index f53936f..c127eeb 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheTwoStepQuery.java
@@ -69,6 +69,9 @@ public class GridCacheTwoStepQuery {
/** */
private List<Integer> extraCaches;
+ /** */
+ private boolean local;
+
/**
* @param originalSql Original query SQL.
* @param schemas Schema names in query.
@@ -229,6 +232,20 @@ public class GridCacheTwoStepQuery {
}
/**
+ * @return {@code True} If query is local.
+ */
+ public boolean isLocal() {
+ return local;
+ }
+
+ /**
+ * @param local Local query flag.
+ */
+ public void local(boolean local) {
+ this.local = local;
+ }
+
+ /**
* @param args New arguments to copy with.
* @return Copy.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index e4b0c1f..2f40d87 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -77,11 +77,13 @@ import org.apache.ignite.internal.processors.cache.CacheObjectContext;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.GridQueryCancel;
import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
@@ -92,6 +94,7 @@ import org.apache.ignite.internal.processors.query.GridQueryIndexing;
import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2DefaultTableEngine;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOffheap;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap;
@@ -187,6 +190,8 @@ import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryTy
import static org.apache.ignite.internal.processors.query.GridQueryIndexType.FULLTEXT;
import static org.apache.ignite.internal.processors.query.GridQueryIndexType.GEO_SPATIAL;
import static org.apache.ignite.internal.processors.query.GridQueryIndexType.SORTED;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.KEY_COL;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.LOCAL;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.PREPARE;
@@ -810,10 +815,22 @@ public class IgniteH2Indexing implements GridQueryIndexing {
removeTable(tbl);
}
- /** {@inheritDoc} */
+ /**
+ * Queries individual fields (generally used by JDBC drivers).
+ *
+ * @param spaceName Space name.
+ * @param qry Query.
+ * @param params Query parameters.
+ * @param filter Space name and key filter.
+ * @param enforceJoinOrder Enforce join order of tables in the query.
+ * @param timeout Query timeout in milliseconds.
+ * @param cancel Query cancel.
+ * @return Query result.
+ * @throws IgniteCheckedException If failed.
+ */
@SuppressWarnings("unchecked")
- @Override public GridQueryFieldsResult queryLocalSqlFields(@Nullable final String spaceName, final String qry,
- @Nullable final Collection<Object> params, final IndexingQueryFilter filters, boolean enforceJoinOrder,
+ public GridQueryFieldsResult queryLocalSqlFields(@Nullable final String spaceName, final String qry,
+ @Nullable final Collection<Object> params, final IndexingQueryFilter filter, boolean enforceJoinOrder,
final int timeout, final GridQueryCancel cancel)
throws IgniteCheckedException {
final Connection conn = connectionForSpace(spaceName);
@@ -833,7 +850,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
fldsQry.setEnforceJoinOrder(enforceJoinOrder);
fldsQry.setTimeout(timeout, TimeUnit.MILLISECONDS);
- return dmlProc.updateLocalSqlFields(spaceName, stmt, fldsQry, filters, cancel);
+ return dmlProc.updateLocalSqlFields(spaceName, stmt, fldsQry, filter, cancel);
}
List<GridQueryFieldMetadata> meta;
@@ -846,7 +863,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
final GridH2QueryContext ctx = new GridH2QueryContext(nodeId, nodeId, 0, LOCAL)
- .filter(filters).distributedJoins(false);
+ .filter(filter).distributedJoinMode(OFF);
return new GridQueryFieldsResultAdapter(meta, null) {
@Override public GridCloseableIterator<List<?>> iterator() throws IgniteCheckedException {
@@ -1099,14 +1116,113 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName,
- final String qry, String alias, @Nullable final Collection<Object> params, GridQueryTypeDescriptor type,
- final IndexingQueryFilter filter) throws IgniteCheckedException {
- final TableDescriptor tbl = tableDescriptor(spaceName, type);
+ @Override public <K, V> QueryCursor<List<?>> queryLocalSqlFields(final GridCacheContext<?, ?> cctx,
+ final SqlFieldsQuery qry, final IndexingQueryFilter filter, final GridQueryCancel cancel)
+ throws IgniteCheckedException {
+
+ if (cctx.config().getQueryParallelism() > 1) {
+ qry.setDistributedJoins(true);
+
+ assert qry.isLocal();
+
+ return queryTwoStep(cctx, qry, cancel);
+ }
+ else {
+ final boolean keepBinary = cctx.keepBinary();
+
+ final String space = cctx.name();
+ final String sql = qry.getSql();
+ final Object[] args = qry.getArgs();
+
+ final GridQueryFieldsResult res = queryLocalSqlFields(space, sql, F.asList(args), filter,
+ qry.isEnforceJoinOrder(), qry.getTimeout(), cancel);
+
+ QueryCursorImpl<List<?>> cursor = new QueryCursorImpl<>(new Iterable<List<?>>() {
+ @Override public Iterator<List<?>> iterator() {
+ try {
+ return new GridQueryCacheObjectsIterator(res.iterator(), cctx, keepBinary);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+ }, cancel);
+
+ cursor.fieldsMeta(res.metaData());
+
+ return cursor;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public <K, V> QueryCursor<Cache.Entry<K,V>> queryLocalSql(final GridCacheContext<?, ?> cctx,
+ final SqlQuery qry, final IndexingQueryFilter filter, final boolean keepBinary) throws IgniteCheckedException {
+ if (cctx.config().getQueryParallelism() > 1) {
+ qry.setDistributedJoins(true);
+
+ assert qry.isLocal();
+
+ return queryTwoStep(cctx, qry);
+ }
+ else {
+ String space = cctx.name();
+ String type = qry.getType();
+ String sqlQry = qry.getSql();
+ String alias = qry.getAlias();
+ Object[] params = qry.getArgs();
+
+ GridQueryCancel cancel = new GridQueryCancel();
+
+ final GridCloseableIterator<IgniteBiTuple<K, V>> i = queryLocalSql(space, sqlQry, alias,
+ F.asList(params), type, filter, cancel);
+
+ return new QueryCursorImpl<Cache.Entry<K, V>>(new Iterable<Cache.Entry<K, V>>() {
+ @Override public Iterator<Cache.Entry<K, V>> iterator() {
+ return new ClIter<Cache.Entry<K, V>>() {
+ @Override public void close() throws Exception {
+ i.close();
+ }
+
+ @Override public boolean hasNext() {
+ return i.hasNext();
+ }
+
+ @Override public Cache.Entry<K, V> next() {
+ IgniteBiTuple<K, V> t = i.next();
+
+ return new CacheEntryImpl<>(
+ (K)cctx.unwrapBinaryIfNeeded(t.get1(), keepBinary, false),
+ (V)cctx.unwrapBinaryIfNeeded(t.get2(), keepBinary, false));
+ }
+
+ @Override public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ }, cancel);
+ }
+ }
+
+ /**
+ * Executes regular query.
+ *
+ * @param spaceName Space name.
+ * @param qry Query.
+ * @param alias Table alias.
+ * @param params Query parameters.
+ * @param type Query return type.
+ * @param filter Space name and key filter.
+ * @return Queried rows.
+ * @throws IgniteCheckedException If failed.
+ */
+ public <K, V> GridCloseableIterator<IgniteBiTuple<K, V>> queryLocalSql(@Nullable String spaceName,
+ final String qry, String alias, @Nullable final Collection<Object> params, String type,
+ final IndexingQueryFilter filter, GridQueryCancel cancel) throws IgniteCheckedException {
+ final TableDescriptor tbl = tableDescriptor(type, spaceName);
if (tbl == null)
- throw new IgniteSQLException("Failed to find SQL table for type: " + type.name(),
+ throw new IgniteSQLException("Failed to find SQL table for type: " + type,
IgniteQueryErrorCode.TABLE_NOT_FOUND);
String sql = generateQuery(qry, alias, tbl);
@@ -1115,7 +1231,8 @@ public class IgniteH2Indexing implements GridQueryIndexing {
setupConnection(conn, false, false);
- GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter).distributedJoins(false));
+ GridH2QueryContext.set(new GridH2QueryContext(nodeId, nodeId, 0, LOCAL).filter(filter)
+ .distributedJoinMode(OFF));
GridRunningQueryInfo run = new GridRunningQueryInfo(qryIdGen.incrementAndGet(), qry, SQL, spaceName,
U.currentTimeMillis(), null, true);
@@ -1123,7 +1240,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
runs.put(run.id(), run);
try {
- ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, null);
+ ResultSet rs = executeSqlQueryWithTimer(spaceName, conn, sql, params, true, 0, cancel);
return new KeyValIterator(rs);
}
@@ -1178,8 +1295,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
fqry.setArgs(qry.getArgs());
fqry.setPageSize(qry.getPageSize());
fqry.setDistributedJoins(qry.isDistributedJoins());
+ fqry.setLocal(qry.isLocal());
- if(qry.getTimeout() > 0)
+ if (qry.getTimeout() > 0)
fqry.setTimeout(qry.getTimeout(), TimeUnit.MILLISECONDS);
final QueryCursor<List<?>> res = queryTwoStep(cctx, fqry, null);
@@ -1234,11 +1352,13 @@ public class IgniteH2Indexing implements GridQueryIndexing {
final boolean distributedJoins = qry.isDistributedJoins() && cctx.isPartitioned();
final boolean grpByCollocated = qry.isCollocated();
+ final DistributedJoinMode distributedJoinMode = distributedJoinMode(qry.isLocal(), distributedJoins);
+
GridCacheTwoStepQuery twoStepQry;
List<GridQueryFieldMetadata> meta;
final TwoStepCachedQueryKey cachedQryKey = new TwoStepCachedQueryKey(space, sqlQry, grpByCollocated,
- distributedJoins, enforceJoinOrder);
+ distributedJoins, enforceJoinOrder, qry.isLocal());
TwoStepCachedQuery cachedQry = twoStepCache.get(cachedQryKey);
if (cachedQry != null) {
@@ -1251,7 +1371,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
setupConnection(c, distributedJoins, enforceJoinOrder);
GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, 0, PREPARE)
- .distributedJoins(distributedJoins));
+ .distributedJoinMode(distributedJoinMode));
PreparedStatement stmt;
@@ -1286,9 +1406,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
GridH2QueryContext.clearThreadLocal();
}
- Prepared prepared = GridSqlQueryParser.prepared((JdbcPreparedStatement) stmt);
+ Prepared prepared = GridSqlQueryParser.prepared((JdbcPreparedStatement)stmt);
- if (qry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery) qry).isQuery() != prepared.isQuery())
+ if (qry instanceof JdbcSqlFieldsQuery && ((JdbcSqlFieldsQuery)qry).isQuery() != prepared.isQuery())
throw new IgniteSQLException("Given statement type does not match that declared by JDBC driver",
IgniteQueryErrorCode.STMT_TYPE_MISMATCH);
@@ -1341,8 +1461,12 @@ public class IgniteH2Indexing implements GridQueryIndexing {
extraCaches = null;
}
+ //Prohibit usage indices with different numbers of segments in same query.
+ checkCacheIndexSegmentation(caches);
+
twoStepQry.caches(caches);
twoStepQry.extraCaches(extraCaches);
+ twoStepQry.local(qry.isLocal());
meta = meta(stmt.getMetaData());
}
@@ -1380,6 +1504,32 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/**
+ * @throws IllegalStateException if segmented indices used with non-segmented indices.
+ */
+ private void checkCacheIndexSegmentation(List<Integer> caches) {
+ if (caches.isEmpty())
+ return; //Nnothing to check
+
+ GridCacheSharedContext sharedContext = ctx.cache().context();
+
+ int expectedParallelism = 0;
+
+ for (int i = 0; i < caches.size(); i++) {
+ GridCacheContext cctx = sharedContext.cacheContext(caches.get(i));
+
+ assert cctx != null;
+
+ if(!cctx.isPartitioned())
+ continue;
+
+ if(expectedParallelism == 0)
+ expectedParallelism = cctx.config().getQueryParallelism();
+ else if (expectedParallelism != 0 && cctx.config().getQueryParallelism() != expectedParallelism)
+ throw new IllegalStateException("Using indexes with different parallelism levels in same query is forbidden.");
+ }
+ }
+
+ /**
* Prepares statement for query.
*
* @param qry Query string.
@@ -1669,7 +1819,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
private void cleanupStatementCache() {
long cur = U.currentTimeMillis();
- for(Iterator<Map.Entry<Thread, StatementCache>> it = stmtCache.entrySet().iterator(); it.hasNext(); ) {
+ for (Iterator<Map.Entry<Thread, StatementCache>> it = stmtCache.entrySet().iterator(); it.hasNext(); ) {
Map.Entry<Thread, StatementCache> entry = it.next();
Thread t = entry.getKey();
@@ -1877,6 +2027,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
for (ClusterNode node : nodes) {
if (node.isLocal()) {
+ if (locNode != null)
+ throw new IllegalStateException();
+
locNode = node;
continue;
@@ -2163,23 +2316,29 @@ public class IgniteH2Indexing implements GridQueryIndexing {
/** */
private final boolean enforceJoinOrder;
+ /** */
+ private final boolean isLocal;
+
/**
* @param space Space.
* @param sql Sql.
* @param grpByCollocated Collocated GROUP BY.
* @param distributedJoins Distributed joins enabled.
* @param enforceJoinOrder Enforce join order of tables.
+ * @param isLocal Query is local flag.
*/
private TwoStepCachedQueryKey(String space,
String sql,
boolean grpByCollocated,
boolean distributedJoins,
- boolean enforceJoinOrder) {
+ boolean enforceJoinOrder,
+ boolean isLocal) {
this.space = space;
this.sql = sql;
this.grpByCollocated = grpByCollocated;
this.distributedJoins = distributedJoins;
this.enforceJoinOrder = enforceJoinOrder;
+ this.isLocal = isLocal;
}
/** {@inheritDoc} */
@@ -2204,7 +2363,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
if (space != null ? !space.equals(that.space) : that.space != null)
return false;
- return sql.equals(that.sql);
+ return isLocal == that.isLocal && sql.equals(that.sql);
}
/** {@inheritDoc} */
@@ -2212,8 +2371,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
int res = space != null ? space.hashCode() : 0;
res = 31 * res + sql.hashCode();
res = 31 * res + (grpByCollocated ? 1 : 0);
- res = 31 * res + (distributedJoins ? 1 : 0);
- res = 31 * res + (enforceJoinOrder ? 1 : 0);
+ res = res + (distributedJoins ? 2 : 0);
+ res = res + (enforceJoinOrder ? 4 : 0);
+ res = res + (isLocal ? 8 : 0);
return res;
}
@@ -2572,7 +2732,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
affCol = null;
// Add primary key index.
- idxs.add(new GridH2TreeIndex("_key_PK", tbl, true,
+ idxs.add(createTreeIndex("_key_PK", tbl, true,
treeIndexColumns(new ArrayList<IndexColumn>(2), keyCol, affCol)));
if (type().valueClass() == String.class) {
@@ -2618,7 +2778,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
cols = treeIndexColumns(cols, keyCol, affCol);
- idxs.add(new GridH2TreeIndex(name, tbl, false, cols));
+ idxs.add(createTreeIndex(name, tbl, false, cols));
}
else if (idx.type() == GEO_SPATIAL)
idxs.add(createH2SpatialIndex(tbl, name, cols.toArray(new IndexColumn[cols.size()])));
@@ -2629,7 +2789,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
// Add explicit affinity key index if nothing alike was found.
if (affCol != null && !affIdxFound) {
- idxs.add(new GridH2TreeIndex("AFFINITY_KEY", tbl, false,
+ idxs.add(createTreeIndex("AFFINITY_KEY", tbl, false,
treeIndexColumns(new ArrayList<IndexColumn>(2), affCol, keyCol)));
}
@@ -2676,6 +2836,22 @@ public class IgniteH2Indexing implements GridQueryIndexing {
throw new IgniteException("Failed to instantiate: " + className, e);
}
}
+
+ /**
+ * @param idxName Index name.
+ * @param tbl Table.
+ * @param pk Primary key flag.
+ * @param columns Index column list.
+ * @return
+ */
+ private Index createTreeIndex(String idxName, GridH2Table tbl, boolean pk, List<IndexColumn> columns) {
+ GridCacheContext<?, ?> cctx = tbl.rowDescriptor().context();
+
+ if (cctx != null && cctx.config().getQueryParallelism() > 1)
+ return new GridH2TreeIndex(idxName, tbl, pk, columns, cctx.config().getQueryParallelism());
+
+ return new GridH2TreeIndex(idxName, tbl, pk, columns, 1);
+ }
}
/**
@@ -2729,6 +2905,13 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/**
+ * Closeable iterator.
+ */
+ private interface ClIter<X> extends AutoCloseable, Iterator<X> {
+ // No-op.
+ }
+
+ /**
* Field descriptor.
*/
static class SqlFieldMetadata implements GridQueryFieldMetadata {
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/DistributedJoinMode.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/DistributedJoinMode.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/DistributedJoinMode.java
new file mode 100644
index 0000000..cc06244
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/DistributedJoinMode.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.opt;
+
+/**
+ * Defines set of distributed join modes.
+ */
+public enum DistributedJoinMode {
+ /**
+ * Distributed joins is disabled. Local joins will be performed instead.
+ */
+ OFF,
+
+ /**
+ * Distributed joins is enabled within local node only.
+ *
+ * NOTE: This mode is used with segmented indices for local sql queries.
+ * As in this case we need to make distributed join across local index segments
+ * and prevent range-queries to other nodes.
+ */
+ LOCAL_ONLY,
+
+ /**
+ * Distributed joins is enabled.
+ */
+ ON;
+
+ /**
+ * @param isLocal Query local flag.
+ * @param distributedJoins Query distributed joins flag.
+ * @return DistributedJoinMode for the query.
+ */
+ public static DistributedJoinMode distributedJoinMode(boolean isLocal, boolean distributedJoins) {
+ return distributedJoins ? (isLocal ? LOCAL_ONLY : ON) : OFF;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index bab219c..131e03b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -81,6 +81,8 @@ import org.jetbrains.annotations.Nullable;
import static java.util.Collections.emptyIterator;
import static java.util.Collections.singletonList;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.LOCAL_ONLY;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.KEY_COL;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.VAL_COL;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2CollocationModel.buildCollocationModel;
@@ -178,6 +180,13 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
/**
+ * @return Index segment ID for current query context.
+ */
+ protected int threadLocalSegment() {
+ return 0;
+ }
+
+ /**
* If the index supports rebuilding it has to creates its own copy.
*
* @return Rebuilt copy.
@@ -252,7 +261,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
// because on run stage reordering of joined tables by Optimizer is explicitly disabled
// and thus multiplier will be always the same, so it will not affect choice of index.
// Query expressions can not be distributed as well.
- if (qctx == null || qctx.type() != PREPARE || !qctx.distributedJoins() || ses.isPreparingQueryExpression())
+ if (qctx == null || qctx.type() != PREPARE || qctx.distributedJoinMode() == OFF || ses.isPreparingQueryExpression())
return GridH2CollocationModel.MULTIPLIER_COLLOCATED;
// We have to clear this cache because normally sub-query plan cost does not depend on anything
@@ -363,7 +372,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
@Override public IndexLookupBatch createLookupBatch(TableFilter filter) {
GridH2QueryContext qctx = GridH2QueryContext.get();
- if (qctx == null || !qctx.distributedJoins() || !getTable().isPartitioned())
+ if (qctx == null || qctx.distributedJoinMode() == OFF || !getTable().isPartitioned())
return null;
IndexColumn affCol = getTable().getAffinityKeyColumn();
@@ -381,9 +390,11 @@ public abstract class GridH2IndexBase extends BaseIndex {
ucast = false;
}
- GridCacheContext<?,?> cctx = getTable().rowDescriptor().context();
+ GridCacheContext<?, ?> cctx = getTable().rowDescriptor().context();
- return new DistributedLookupBatch(cctx, ucast, affColId);
+ boolean isLocal = qctx.distributedJoinMode() == LOCAL_ONLY;
+
+ return new DistributedLookupBatch(cctx, ucast, affColId, isLocal);
}
/**
@@ -437,18 +448,18 @@ public abstract class GridH2IndexBase extends BaseIndex {
* @param node Requesting node.
* @param msg Request message.
*/
- private void onIndexRangeRequest(ClusterNode node, GridH2IndexRangeRequest msg) {
- GridH2QueryContext qctx = GridH2QueryContext.get(kernalContext().localNodeId(),
- msg.originNodeId(),
- msg.queryId(),
- MAP);
-
+ private void onIndexRangeRequest(final ClusterNode node, final GridH2IndexRangeRequest msg) {
GridH2IndexRangeResponse res = new GridH2IndexRangeResponse();
res.originNodeId(msg.originNodeId());
res.queryId(msg.queryId());
+ res.originSegmentId(msg.originSegmentId());
+ res.segment(msg.segment());
res.batchLookupId(msg.batchLookupId());
+ GridH2QueryContext qctx = GridH2QueryContext.get(kernalContext().localNodeId(), msg.originNodeId(),
+ msg.queryId(), msg.originSegmentId(), MAP);
+
if (qctx == null)
res.status(STATUS_NOT_FOUND);
else {
@@ -461,11 +472,11 @@ public abstract class GridH2IndexBase extends BaseIndex {
assert !msg.bounds().isEmpty() : "empty bounds";
- src = new RangeSource(msg.bounds(), snapshot0, qctx.filter());
+ src = new RangeSource(msg.bounds(), msg.segment(), snapshot0, qctx.filter());
}
else {
// This is request to fetch next portion of data.
- src = qctx.getSource(node.id(), msg.batchLookupId());
+ src = qctx.getSource(node.id(), msg.segment(), msg.batchLookupId());
assert src != null;
}
@@ -491,11 +502,11 @@ public abstract class GridH2IndexBase extends BaseIndex {
if (src.hasMoreRows()) {
// Save source for future fetches.
if (msg.bounds() != null)
- qctx.putSource(node.id(), msg.batchLookupId(), src);
+ qctx.putSource(node.id(), msg.segment(), msg.batchLookupId(), src);
}
else if (msg.bounds() == null) {
// Drop saved source.
- qctx.putSource(node.id(), msg.batchLookupId(), null);
+ qctx.putSource(node.id(), msg.segment(), msg.batchLookupId(), null);
}
assert !ranges.isEmpty();
@@ -520,17 +531,17 @@ public abstract class GridH2IndexBase extends BaseIndex {
*/
private void onIndexRangeResponse(ClusterNode node, GridH2IndexRangeResponse msg) {
GridH2QueryContext qctx = GridH2QueryContext.get(kernalContext().localNodeId(),
- msg.originNodeId(), msg.queryId(), MAP);
+ msg.originNodeId(), msg.queryId(), msg.originSegmentId(), MAP);
if (qctx == null)
return;
- Map<ClusterNode, RangeStream> streams = qctx.getStreams(msg.batchLookupId());
+ Map<SegmentKey, RangeStream> streams = qctx.getStreams(msg.batchLookupId());
if (streams == null)
return;
- RangeStream stream = streams.get(node);
+ RangeStream stream = streams.get(new SegmentKey(node, msg.segment()));
assert stream != null;
@@ -549,47 +560,69 @@ public abstract class GridH2IndexBase extends BaseIndex {
/**
* @param qctx Query context.
* @param batchLookupId Batch lookup ID.
+ * @param segmentId Segment ID.
* @return Index range request.
*/
- private static GridH2IndexRangeRequest createRequest(GridH2QueryContext qctx, int batchLookupId) {
+ private static GridH2IndexRangeRequest createRequest(GridH2QueryContext qctx, int batchLookupId, int segmentId) {
GridH2IndexRangeRequest req = new GridH2IndexRangeRequest();
req.originNodeId(qctx.originNodeId());
req.queryId(qctx.queryId());
+ req.originSegmentId(qctx.segment());
+ req.segment(segmentId);
req.batchLookupId(batchLookupId);
return req;
}
+
/**
* @param qctx Query context.
* @param cctx Cache context.
+ * @param isLocalQry Local query flag.
* @return Collection of nodes for broadcasting.
*/
- private List<ClusterNode> broadcastNodes(GridH2QueryContext qctx, GridCacheContext<?,?> cctx) {
+ private List<SegmentKey> broadcastSegments(GridH2QueryContext qctx, GridCacheContext<?, ?> cctx, boolean isLocalQry) {
Map<UUID, int[]> partMap = qctx.partitionsMap();
- List<ClusterNode> res;
+ List<ClusterNode> nodes;
+
+ if (isLocalQry) {
+ if (partMap != null && !partMap.containsKey(cctx.localNodeId()))
+ return Collections.<SegmentKey>emptyList(); // Prevent remote index call for local queries.
- if (partMap == null)
- res = new ArrayList<>(CU.affinityNodes(cctx, qctx.topologyVersion()));
+ nodes = Collections.singletonList(cctx.localNode());
+ }
else {
- res = new ArrayList<>(partMap.size());
+ if (partMap == null)
+ nodes = new ArrayList<>(CU.affinityNodes(cctx, qctx.topologyVersion()));
+ else {
+ nodes = new ArrayList<>(partMap.size());
- GridKernalContext ctx = kernalContext();
+ GridKernalContext ctx = kernalContext();
- for (UUID nodeId : partMap.keySet()) {
- ClusterNode node = ctx.discovery().node(nodeId);
+ for (UUID nodeId : partMap.keySet()) {
+ ClusterNode node = ctx.discovery().node(nodeId);
- if (node == null)
- throw new GridH2RetryException("Failed to find node.");
+ if (node == null)
+ throw new GridH2RetryException("Failed to find node.");
- res.add(node);
+ nodes.add(node);
+ }
}
+
+ if (F.isEmpty(nodes))
+ throw new GridH2RetryException("Failed to collect affinity nodes.");
}
- if (F.isEmpty(res))
- throw new GridH2RetryException("Failed to collect affinity nodes.");
+ int segmentsCount = segmentsCount();
+
+ List<SegmentKey> res = new ArrayList<>(nodes.size() * segmentsCount);
+
+ for (ClusterNode node : nodes) {
+ for (int seg = 0; seg < segmentsCount; seg++)
+ res.add(new SegmentKey(node, seg));
+ }
return res;
}
@@ -598,26 +631,81 @@ public abstract class GridH2IndexBase extends BaseIndex {
* @param cctx Cache context.
* @param qctx Query context.
* @param affKeyObj Affinity key.
- * @return Cluster nodes or {@code null} if affinity key is a null value.
+ * @param isLocalQry Local query flag.
+ * @return Segment key for Affinity key.
*/
- private ClusterNode rangeNode(GridCacheContext<?,?> cctx, GridH2QueryContext qctx, Object affKeyObj) {
+ private SegmentKey rangeSegment(GridCacheContext<?, ?> cctx, GridH2QueryContext qctx, Object affKeyObj, boolean isLocalQry) {
assert affKeyObj != null && affKeyObj != EXPLICIT_NULL : affKeyObj;
ClusterNode node;
- if (qctx.partitionsMap() != null) {
- // If we have explicit partitions map, we have to use it to calculate affinity node.
- UUID nodeId = qctx.nodeForPartition(cctx.affinity().partition(affKeyObj), cctx);
+ int partition = cctx.affinity().partition(affKeyObj);
+
+ if (isLocalQry) {
+ if (qctx.partitionsMap() != null) {
+ // If we have explicit partitions map, we have to use it to calculate affinity node.
+ UUID nodeId = qctx.nodeForPartition(partition, cctx);
+
+ if(!cctx.localNodeId().equals(nodeId))
+ return null; // Prevent remote index call for local queries.
+ }
+
+ if (!cctx.affinity().primaryByKey(cctx.localNode(), partition, qctx.topologyVersion()))
+ return null;
+
+ node = cctx.localNode();
+ }
+ else{
+ if (qctx.partitionsMap() != null) {
+ // If we have explicit partitions map, we have to use it to calculate affinity node.
+ UUID nodeId = qctx.nodeForPartition(partition, cctx);
node = cctx.discovery().node(nodeId);
}
else // Get primary node for current topology version.
node = cctx.affinity().primaryByKey(affKeyObj, qctx.topologyVersion());
- if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
- throw new GridH2RetryException("Failed to find node.");
+ if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
+ throw new GridH2RetryException("Failed to find node.");
+ }
+
+ return new SegmentKey(node, segment(partition));
+ }
+
+ /** */
+ protected class SegmentKey {
+ /** */
+ final ClusterNode node;
+
+ /** */
+ final int segmentId;
+
+ SegmentKey(ClusterNode node, int segmentId) {
+ assert node != null;
+
+ this.node = node;
+ this.segmentId = segmentId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ SegmentKey key = (SegmentKey)o;
+
+ return segmentId == key.segmentId && node.id().equals(key.node.id());
- return node;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int result = node.hashCode();
+ result = 31 * result + segmentId;
+ return result;
+ }
}
/**
@@ -740,6 +828,20 @@ public abstract class GridH2IndexBase extends BaseIndex {
return database.createRow(vals0, MEMORY_CALCULATE);
}
+ /** @return Index segments count. */
+ protected int segmentsCount() {
+ return 1;
+ }
+
+ /**
+ * @param partition Partition idx.
+ * @return Segment ID for given key
+ */
+ protected int segment(int partition) {
+ return 0;
+ }
+
+
/**
* Simple cursor from a single node.
*/
@@ -752,14 +854,14 @@ public abstract class GridH2IndexBase extends BaseIndex {
/**
* @param rangeId Range ID.
- * @param nodes Remote nodes.
+ * @param keys Remote index segment keys.
* @param rangeStreams Range streams.
*/
- private UnicastCursor(int rangeId, Collection<ClusterNode> nodes, Map<ClusterNode,RangeStream> rangeStreams) {
- assert nodes.size() == 1;
+ UnicastCursor(int rangeId, List<SegmentKey> keys, Map<SegmentKey, RangeStream> rangeStreams) {
+ assert keys.size() == 1;
this.rangeId = rangeId;
- this.stream = rangeStreams.get(F.first(nodes));
+ this.stream = rangeStreams.get(F.first(keys));
assert stream != null;
}
@@ -803,20 +905,19 @@ public abstract class GridH2IndexBase extends BaseIndex {
/**
* @param rangeId Range ID.
- * @param nodes Remote nodes.
+ * @param segmentKeys Remote nodes.
* @param rangeStreams Range streams.
*/
- private BroadcastCursor(int rangeId, Collection<ClusterNode> nodes, Map<ClusterNode,RangeStream> rangeStreams) {
- assert nodes.size() > 1;
+ BroadcastCursor(int rangeId, Collection<SegmentKey> segmentKeys, Map<SegmentKey, RangeStream> rangeStreams) {
this.rangeId = rangeId;
- streams = new RangeStream[nodes.size()];
+ streams = new RangeStream[segmentKeys.size()];
int i = 0;
- for (ClusterNode node : nodes) {
- RangeStream stream = rangeStreams.get(node);
+ for (SegmentKey segmentKey : segmentKeys) {
+ RangeStream stream = rangeStreams.get(segmentKey);
assert stream != null;
@@ -928,16 +1029,19 @@ public abstract class GridH2IndexBase extends BaseIndex {
final int affColId;
/** */
+ private final boolean localQuery;
+
+ /** */
GridH2QueryContext qctx;
/** */
int batchLookupId;
/** */
- Map<ClusterNode, RangeStream> rangeStreams = Collections.emptyMap();
+ Map<SegmentKey, RangeStream> rangeStreams = Collections.emptyMap();
/** */
- List<ClusterNode> broadcastNodes;
+ List<SegmentKey> broadcastSegments;
/** */
List<Future<Cursor>> res = Collections.emptyList();
@@ -952,11 +1056,13 @@ public abstract class GridH2IndexBase extends BaseIndex {
* @param cctx Cache Cache context.
* @param ucast Unicast or broadcast query.
* @param affColId Affinity column ID.
+ * @param localQuery Local query flag.
*/
- private DistributedLookupBatch(GridCacheContext<?,?> cctx, boolean ucast, int affColId) {
+ DistributedLookupBatch(GridCacheContext<?, ?> cctx, boolean ucast, int affColId, boolean localQuery) {
this.cctx = cctx;
this.ucast = ucast;
this.affColId = affColId;
+ this.localQuery = localQuery;
}
/**
@@ -1028,7 +1134,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
Object affKey = affColId == -1 ? null : getAffinityKey(firstRow, lastRow);
- List<ClusterNode> nodes;
+ List<SegmentKey> segmentKeys;
Future<Cursor> fut;
if (affKey != null) {
@@ -1036,17 +1142,20 @@ public abstract class GridH2IndexBase extends BaseIndex {
if (affKey == EXPLICIT_NULL) // Affinity key is explicit null, we will not find anything.
return false;
- nodes = F.asList(rangeNode(cctx, qctx, affKey));
+ segmentKeys = F.asList(rangeSegment(cctx, qctx, affKey, localQuery));
}
else {
// Affinity key is not provided or is not the same in upper and lower bounds, we have to broadcast.
- if (broadcastNodes == null)
- broadcastNodes = broadcastNodes(qctx, cctx);
+ if (broadcastSegments == null)
+ broadcastSegments = broadcastSegments(qctx, cctx, localQuery);
- nodes = broadcastNodes;
+ segmentKeys = broadcastSegments;
}
- assert !F.isEmpty(nodes) : nodes;
+ if (localQuery && segmentKeys.isEmpty())
+ return false; // Nothing to do
+
+ assert !F.isEmpty(segmentKeys) : segmentKeys;
final int rangeId = res.size();
@@ -1058,21 +1167,21 @@ public abstract class GridH2IndexBase extends BaseIndex {
GridH2RowRangeBounds rangeBounds = rangeBounds(rangeId, first, last);
// Add range to every message of every participating node.
- for (int i = 0; i < nodes.size(); i++) {
- ClusterNode node = nodes.get(i);
- assert node != null;
+ for (int i = 0; i < segmentKeys.size(); i++) {
+ SegmentKey segmentKey = segmentKeys.get(i);
+ assert segmentKey != null;
- RangeStream stream = rangeStreams.get(node);
+ RangeStream stream = rangeStreams.get(segmentKey);
List<GridH2RowRangeBounds> bounds;
if (stream == null) {
- stream = new RangeStream(qctx, node);
+ stream = new RangeStream(qctx, segmentKey.node);
- stream.req = createRequest(qctx, batchLookupId);
+ stream.req = createRequest(qctx, batchLookupId, segmentKey.segmentId);
stream.req.bounds(bounds = new ArrayList<>());
- rangeStreams.put(node, stream);
+ rangeStreams.put(segmentKey, stream);
}
else
bounds = stream.req.bounds();
@@ -1084,9 +1193,9 @@ public abstract class GridH2IndexBase extends BaseIndex {
batchFull = true;
}
- fut = new DoneFuture<>(nodes.size() == 1 ?
- new UnicastCursor(rangeId, nodes, rangeStreams) :
- new BroadcastCursor(rangeId, nodes, rangeStreams));
+ fut = new DoneFuture<>(segmentKeys.size() == 1 ?
+ new UnicastCursor(rangeId, segmentKeys, rangeStreams) :
+ new BroadcastCursor(rangeId, segmentKeys, rangeStreams));
res.add(fut);
@@ -1138,7 +1247,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
batchLookupId = 0;
rangeStreams = Collections.emptyMap();
- broadcastNodes = null;
+ broadcastSegments = null;
batchFull = false;
findCalled = false;
res = Collections.emptyList();
@@ -1244,7 +1353,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
if (remainingRanges > 0) {
if (req.bounds() != null)
- req = createRequest(qctx, req.batchLookupId());
+ req = createRequest(qctx, req.batchLookupId(), req.segment());
// Prefetch next page.
send(singletonList(node), req);
@@ -1366,6 +1475,9 @@ public abstract class GridH2IndexBase extends BaseIndex {
final ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree;
/** */
+ private final int segment;
+
+ /** */
final IndexingQueryFilter filter;
/**
@@ -1375,9 +1487,11 @@ public abstract class GridH2IndexBase extends BaseIndex {
*/
RangeSource(
Iterable<GridH2RowRangeBounds> bounds,
+ int segment,
ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree,
IndexingQueryFilter filter
) {
+ this.segment = segment;
this.filter = filter;
this.tree = tree;
boundsIter = bounds.iterator();
@@ -1435,7 +1549,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
SearchRow first = toSearchRow(bounds.first());
SearchRow last = toSearchRow(bounds.last());
- ConcurrentNavigableMap<GridSearchRowPointer,GridH2Row> t = tree != null ? tree : treeForRead();
+ ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> t = tree != null ? tree : treeForRead(segment);
curRange = doFind0(t, first, true, last, filter);
@@ -1452,9 +1566,10 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
/**
- * @return Snapshot for current thread if there is one.
+ * @param segment Segment Id.
+ * @return Snapshot for requested segment if there is one.
*/
- protected ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> treeForRead() {
+ protected ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> treeForRead(int segment) {
throw new UnsupportedOperationException();
}
@@ -1505,7 +1620,8 @@ public abstract class GridH2IndexBase extends BaseIndex {
this.fltr = qryFilter.forSpace(spaceName);
this.isValRequired = qryFilter.isValueRequired();
- } else {
+ }
+ else {
this.fltr = null;
this.isValRequired = false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2QueryContext.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2QueryContext.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2QueryContext.java
index 19ea2b2..a7ee0dc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2QueryContext.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2QueryContext.java
@@ -32,6 +32,7 @@ import org.apache.ignite.spi.indexing.IndexingQueryFilter;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentHashMap8;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.MAP;
/**
@@ -79,7 +80,7 @@ public class GridH2QueryContext {
private UUID[] partsNodes;
/** */
- private boolean distributedJoins;
+ private DistributedJoinMode distributedJoinMode;
/** */
private int pageSize;
@@ -94,7 +95,22 @@ public class GridH2QueryContext {
* @param type Query type.
*/
public GridH2QueryContext(UUID locNodeId, UUID nodeId, long qryId, GridH2QueryType type) {
- key = new Key(locNodeId, nodeId, qryId, type);
+ assert type != MAP;
+
+ key = new Key(locNodeId, nodeId, qryId, 0, type);
+ }
+
+ /**
+ * @param locNodeId Local node ID.
+ * @param nodeId The node who initiated the query.
+ * @param qryId The query ID.
+ * @param segmentId Index segment ID.
+ * @param type Query type.
+ */
+ public GridH2QueryContext(UUID locNodeId, UUID nodeId, long qryId, int segmentId, GridH2QueryType type) {
+ assert segmentId == 0 || type == MAP;
+
+ key = new Key(locNodeId, nodeId, qryId, segmentId, type);
}
/**
@@ -133,20 +149,20 @@ public class GridH2QueryContext {
}
/**
- * @param distributedJoins Distributed joins can be run in this query.
+ * @param distributedJoinMode Distributed join mode.
* @return {@code this}.
*/
- public GridH2QueryContext distributedJoins(boolean distributedJoins) {
- this.distributedJoins = distributedJoins;
+ public GridH2QueryContext distributedJoinMode(DistributedJoinMode distributedJoinMode) {
+ this.distributedJoinMode = distributedJoinMode;
return this;
}
/**
- * @return {@code true} If distributed joins can be run in this query.
+ * @return Distributed join mode.
*/
- public boolean distributedJoins() {
- return distributedJoins;
+ public DistributedJoinMode distributedJoinMode() {
+ return distributedJoinMode;
}
/**
@@ -226,6 +242,11 @@ public class GridH2QueryContext {
return nodeIds[p];
}
+ /** @return index segment ID. */
+ public int segment() {
+ return key.segmentId;
+ }
+
/**
* @param idxId Index ID.
* @param snapshot Index snapshot.
@@ -303,11 +324,12 @@ public class GridH2QueryContext {
/**
* @param ownerId Owner node ID.
+ * @param segmentId Index segment ID.
* @param batchLookupId Batch lookup ID.
* @param src Range source.
*/
- public synchronized void putSource(UUID ownerId, int batchLookupId, Object src) {
- SourceKey srcKey = new SourceKey(ownerId, batchLookupId);
+ public synchronized void putSource(UUID ownerId, int segmentId, int batchLookupId, Object src) {
+ SourceKey srcKey = new SourceKey(ownerId, segmentId, batchLookupId);
if (src != null) {
if (sources == null)
@@ -321,15 +343,16 @@ public class GridH2QueryContext {
/**
* @param ownerId Owner node ID.
+ * @param segmentId Index segment ID.
* @param batchLookupId Batch lookup ID.
* @return Range source.
*/
@SuppressWarnings("unchecked")
- public synchronized <T> T getSource(UUID ownerId, int batchLookupId) {
+ public synchronized <T> T getSource(UUID ownerId, int segmentId, int batchLookupId) {
if (sources == null)
return null;
- return (T)sources.get(new SourceKey(ownerId, batchLookupId));
+ return (T)sources.get(new SourceKey(ownerId, segmentId, batchLookupId));
}
/**
@@ -356,7 +379,7 @@ public class GridH2QueryContext {
assert qctx.get() == null;
// We need MAP query context to be available to other threads to run distributed joins.
- if (x.key.type == MAP && x.distributedJoins() && qctxs.putIfAbsent(x.key, x) != null)
+ if (x.key.type == MAP && x.distributedJoinMode() != OFF && qctxs.putIfAbsent(x.key, x) != null)
throw new IllegalStateException("Query context is already set.");
qctx.set(x);
@@ -381,7 +404,14 @@ public class GridH2QueryContext {
* @return {@code True} if context was found.
*/
public static boolean clear(UUID locNodeId, UUID nodeId, long qryId, GridH2QueryType type) {
- return doClear(new Key(locNodeId, nodeId, qryId, type), false);
+ boolean res = false;
+
+ for (Key key : qctxs.keySet()) {
+ if (key.locNodeId.equals(locNodeId) && key.nodeId.equals(nodeId) && key.qryId == qryId && key.type == type)
+ res |= doClear(new Key(locNodeId, nodeId, qryId, key.segmentId, type), false);
+ }
+
+ return res;
}
/**
@@ -463,6 +493,7 @@ public class GridH2QueryContext {
* @param locNodeId Local node ID.
* @param nodeId The node who initiated the query.
* @param qryId The query ID.
+ * @param segmentId Index segment ID.
* @param type Query type.
* @return Query context.
*/
@@ -470,9 +501,10 @@ public class GridH2QueryContext {
UUID locNodeId,
UUID nodeId,
long qryId,
+ int segmentId,
GridH2QueryType type
) {
- return qctxs.get(new Key(locNodeId, nodeId, qryId, type));
+ return qctxs.get(new Key(locNodeId, nodeId, qryId, segmentId, type));
}
/**
@@ -528,15 +560,19 @@ public class GridH2QueryContext {
private final long qryId;
/** */
+ private final int segmentId;
+
+ /** */
private final GridH2QueryType type;
/**
* @param locNodeId Local node ID.
* @param nodeId The node who initiated the query.
* @param qryId The query ID.
+ * @param segmentId Index segment ID.
* @param type Query type.
*/
- private Key(UUID locNodeId, UUID nodeId, long qryId, GridH2QueryType type) {
+ private Key(UUID locNodeId, UUID nodeId, long qryId, int segmentId, GridH2QueryType type) {
assert locNodeId != null;
assert nodeId != null;
assert type != null;
@@ -544,6 +580,7 @@ public class GridH2QueryContext {
this.locNodeId = locNodeId;
this.nodeId = nodeId;
this.qryId = qryId;
+ this.segmentId = segmentId;
this.type = type;
}
@@ -568,6 +605,7 @@ public class GridH2QueryContext {
res = 31 * res + nodeId.hashCode();
res = 31 * res + (int)(qryId ^ (qryId >>> 32));
res = 31 * res + type.hashCode();
+ res = 31 * res + segmentId;
return res;
}
@@ -586,14 +624,19 @@ public class GridH2QueryContext {
UUID ownerId;
/** */
+ int segmentId;
+
+ /** */
int batchLookupId;
/**
* @param ownerId Owner node ID.
+ * @param segmentId Index segment ID.
* @param batchLookupId Batch lookup ID.
*/
- SourceKey(UUID ownerId, int batchLookupId) {
+ SourceKey(UUID ownerId, int segmentId, int batchLookupId) {
this.ownerId = ownerId;
+ this.segmentId = segmentId;
this.batchLookupId = batchLookupId;
}
@@ -601,12 +644,15 @@ public class GridH2QueryContext {
@Override public boolean equals(Object o) {
SourceKey srcKey = (SourceKey)o;
- return batchLookupId == srcKey.batchLookupId && ownerId.equals(srcKey.ownerId);
+ return batchLookupId == srcKey.batchLookupId && segmentId == srcKey.segmentId &&
+ ownerId.equals(srcKey.ownerId);
}
/** {@inheritDoc} */
@Override public int hashCode() {
- return 31 * ownerId.hashCode() + batchLookupId;
+ int hash = ownerId.hashCode();
+ hash = 31 * hash + segmentId;
+ return 31 * hash + batchLookupId;
}
}
}
[14/50] [abbrv] ignite git commit: Implemented.
Posted by av...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 914e0da..0829df0 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -17,12 +17,15 @@
package org.apache.ignite.internal.processors.query.h2.opt;
+import java.lang.reflect.Field;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.util.GridEmptyIterator;
import org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap;
import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeGuard;
@@ -43,18 +46,36 @@ import org.h2.value.Value;
import org.jetbrains.annotations.Nullable;
/**
- * Base class for snapshotable tree indexes.
+ * Base class for snapshotable segmented tree indexes.
*/
@SuppressWarnings("ComparatorNotSerializable")
public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridSearchRowPointer> {
+
/** */
- private final ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree;
+ private static Field KEY_FIELD;
+
+ /** */
+ static {
+ try {
+ KEY_FIELD = GridH2AbstractKeyValueRow.class.getDeclaredField("key");
+ KEY_FIELD.setAccessible(true);
+ }
+ catch (NoSuchFieldException e) {
+ KEY_FIELD = null;
+ }
+ }
+
+ /** Index segments. */
+ private final ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row>[] segments;
/** */
private final boolean snapshotEnabled;
+ /** */
+ private final GridH2RowDescriptor desc;
+
/**
- * Constructor with index initialization.
+ * Constructor with index initialization. Creates index with single segment.
*
* @param name Index name.
* @param tbl Table.
@@ -63,35 +84,59 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
*/
@SuppressWarnings("unchecked")
public GridH2TreeIndex(String name, GridH2Table tbl, boolean pk, List<IndexColumn> colsList) {
+ this(name, tbl, pk, colsList, 1);
+ }
+
+ /**
+ * Constructor with index initialization.
+ *
+ * @param name Index name.
+ * @param tbl Table.
+ * @param pk If this index is primary key.
+ * @param colsList Index columns list.
+ * @param segmentsCnt Number of segments.
+ */
+ @SuppressWarnings("unchecked")
+ public GridH2TreeIndex(String name, GridH2Table tbl, boolean pk, List<IndexColumn> colsList, int segmentsCnt) {
+ assert segmentsCnt > 0 : segmentsCnt;
+
IndexColumn[] cols = colsList.toArray(new IndexColumn[colsList.size()]);
IndexColumn.mapColumns(cols, tbl);
+ desc = tbl.rowDescriptor();
+
initBaseIndex(tbl, 0, name, cols,
pk ? IndexType.createPrimaryKey(false, false) : IndexType.createNonUnique(false, false, false));
+ segments = new ConcurrentNavigableMap[segmentsCnt];
+
final GridH2RowDescriptor desc = tbl.rowDescriptor();
if (desc == null || desc.memory() == null) {
snapshotEnabled = desc == null || desc.snapshotableIndex();
if (snapshotEnabled) {
- tree = new SnapTreeMap<GridSearchRowPointer, GridH2Row>(this) {
- @Override protected void afterNodeUpdate_nl(Node<GridSearchRowPointer, GridH2Row> node, Object val) {
- if (val != null)
- node.key = (GridSearchRowPointer)val;
- }
+ for (int i = 0; i < segmentsCnt; i++) {
+ segments[i] = new SnapTreeMap<GridSearchRowPointer, GridH2Row>(this) {
+ @Override
+ protected void afterNodeUpdate_nl(Node<GridSearchRowPointer, GridH2Row> node, Object val) {
+ if (val != null)
+ node.key = (GridSearchRowPointer)val;
+ }
- @Override protected Comparable<? super GridSearchRowPointer> comparable(Object key) {
- if (key instanceof ComparableRow)
- return (Comparable<? super SearchRow>)key;
+ @Override protected Comparable<? super GridSearchRowPointer> comparable(Object key) {
+ if (key instanceof ComparableRow)
+ return (Comparable<? super SearchRow>)key;
- return super.comparable(key);
- }
- };
+ return super.comparable(key);
+ }
+ };
+ }
}
else {
- tree = new ConcurrentSkipListMap<>(
+ for (int i = 0; i < segmentsCnt; i++) {
+ segments[i] = new ConcurrentSkipListMap<>(
new Comparator<GridSearchRowPointer>() {
@Override public int compare(GridSearchRowPointer o1, GridSearchRowPointer o2) {
if (o1 instanceof ComparableRow)
@@ -103,7 +148,8 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
return compareRows(o1, o2);
}
}
- );
+ );
+ }
}
}
else {
@@ -111,28 +157,30 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
snapshotEnabled = true;
- tree = new GridOffHeapSnapTreeMap<GridSearchRowPointer, GridH2Row>(desc, desc, desc.memory(), desc.guard(), this) {
- @Override protected void afterNodeUpdate_nl(long node, GridH2Row val) {
- final long oldKey = keyPtr(node);
+ for (int i = 0; i < segmentsCnt; i++) {
+ segments[i] = new GridOffHeapSnapTreeMap<GridSearchRowPointer, GridH2Row>(desc, desc, desc.memory(), desc.guard(), this) {
+ @Override protected void afterNodeUpdate_nl(long node, GridH2Row val) {
+ final long oldKey = keyPtr(node);
- if (val != null) {
- key(node, val);
+ if (val != null) {
+ key(node, val);
- guard.finalizeLater(new Runnable() {
- @Override public void run() {
- desc.createPointer(oldKey).decrementRefCount();
- }
- });
+ guard.finalizeLater(new Runnable() {
+ @Override public void run() {
+ desc.createPointer(oldKey).decrementRefCount();
+ }
+ });
+ }
}
- }
- @Override protected Comparable<? super GridSearchRowPointer> comparable(Object key) {
- if (key instanceof ComparableRow)
- return (Comparable<? super SearchRow>)key;
+ @Override protected Comparable<? super GridSearchRowPointer> comparable(Object key) {
+ if (key instanceof ComparableRow)
+ return (Comparable<? super SearchRow>)key;
- return super.comparable(key);
- }
- };
+ return super.comparable(key);
+ }
+ };
+ }
}
initDistributedJoinMessaging(tbl);
@@ -142,20 +190,24 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
@Override protected Object doTakeSnapshot() {
assert snapshotEnabled;
+ int seg = threadLocalSegment();
+
+ ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> tree = segments[seg];
+
return tree instanceof SnapTreeMap ?
((SnapTreeMap)tree).clone() :
((GridOffHeapSnapTreeMap)tree).clone();
}
/** {@inheritDoc} */
- protected final ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> treeForRead() {
+ protected ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> treeForRead(int seg) {
if (!snapshotEnabled)
- return tree;
+ return segments[seg];
ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> res = threadLocalSnapshot();
if (res == null)
- res = tree;
+ return segments[seg];
return res;
}
@@ -164,19 +216,37 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
@Override public void destroy() {
assert threadLocalSnapshot() == null;
- if (tree instanceof AutoCloseable)
- U.closeQuiet((AutoCloseable)tree);
+ for (int i = 0; i < segments.length; i++) {
+ if (segments[i] instanceof AutoCloseable)
+ U.closeQuiet((AutoCloseable)segments[i]);
+ }
super.destroy();
}
+
+ /** {@inheritDoc} */
+ protected int threadLocalSegment() {
+ GridH2QueryContext qctx = GridH2QueryContext.get();
+
+ if(segments.length == 1)
+ return 0;
+
+ if(qctx == null)
+ throw new IllegalStateException("GridH2QueryContext is not initialized.");
+
+ return qctx.segment();
+ }
+
/** {@inheritDoc} */
@Override public long getRowCount(@Nullable Session ses) {
IndexingQueryFilter f = threadLocalFilter();
+ int seg = threadLocalSegment();
+
// Fast path if we don't need to perform any filtering.
if (f == null || f.forSpace((getTable()).spaceName()) == null)
- return treeForRead().size();
+ return treeForRead(seg).size();
Iterator<GridH2Row> iter = doFind(null, false, null);
@@ -255,7 +325,9 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
* @return Row.
*/
public GridH2Row findOne(GridSearchRowPointer row) {
- return tree.get(row);
+ int seg = threadLocalSegment();
+
+ return segments[seg].get(row);
}
/**
@@ -268,7 +340,9 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
*/
@SuppressWarnings("unchecked")
private Iterator<GridH2Row> doFind(@Nullable SearchRow first, boolean includeFirst, @Nullable SearchRow last) {
- ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> t = treeForRead();
+ int seg = threadLocalSegment();
+
+ ConcurrentNavigableMap<GridSearchRowPointer, GridH2Row> t = treeForRead(seg);
return doFind0(t, first, includeFirst, last, threadLocalFilter());
}
@@ -359,12 +433,68 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
/** {@inheritDoc} */
@Override public GridH2Row put(GridH2Row row) {
- return tree.put(row, row);
+ int seg = segment(row);
+
+ return segments[seg].put(row, row);
}
/** {@inheritDoc} */
@Override public GridH2Row remove(SearchRow row) {
- return tree.remove(comparable(row, 0));
+ GridSearchRowPointer comparable = comparable(row, 0);
+
+ int seg = segment(row);
+
+ return segments[seg].remove(comparable);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int segmentsCount() {
+ return segments.length;
+ }
+
+ /**
+ * @param partition Parttition idx.
+ * @return index currentSegment Id for given key
+ */
+ protected int segment(int partition) {
+ return partition % segments.length;
+ }
+
+ /**
+ * @param row
+ * @return index currentSegment Id for given row
+ */
+ private int segment(SearchRow row) {
+ assert row != null;
+
+ CacheObject key;
+
+ if (desc != null && desc.context() != null) {
+ GridCacheContext<?, ?> ctx = desc.context();
+
+ assert ctx != null;
+
+ if (row instanceof GridH2AbstractKeyValueRow && KEY_FIELD != null) {
+ try {
+ Object o = KEY_FIELD.get(row);
+
+ if (o instanceof CacheObject)
+ key = (CacheObject)o;
+ else
+ key = ctx.toCacheKeyObject(o);
+
+ }
+ catch (IllegalAccessException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ else
+ key = ctx.toCacheKeyObject(row.getValue(0));
+
+ return segment(ctx.affinity().partition(key));
+ }
+ else
+ return 0;
}
/**
@@ -463,18 +593,20 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
IndexColumn[] cols = getIndexColumns();
GridH2TreeIndex idx = new GridH2TreeIndex(getName(), getTable(),
- getIndexType().isUnique(), F.asList(cols));
+ getIndexType().isUnique(), F.asList(cols), segments.length);
Thread thread = Thread.currentThread();
- long i = 0;
+ long j = 0;
- for (GridH2Row row : tree.values()) {
- // Check for interruptions every 1000 iterations.
- if (++i % 1000 == 0 && thread.isInterrupted())
- throw new InterruptedException();
+ for (int i = 0; i < segments.length; i++) {
+ for (GridH2Row row : segments[i].values()) {
+ // Check for interruptions every 1000 iterations.
+ if ((++j & 1023) == 0 && thread.isInterrupted())
+ throw new InterruptedException();
- idx.tree.put(row, row);
+ idx.put(row);
+ }
}
return idx;
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index ac1a6a6..5027c9a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReferenceArray;
import javax.cache.CacheException;
@@ -56,6 +57,7 @@ import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshalla
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.query.GridQueryCancel;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RetryException;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
@@ -84,6 +86,8 @@ import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.MAP;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REPLICATED;
import static org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor.QUERY_POOL;
@@ -161,8 +165,7 @@ public class GridMapQueryExecutor {
if (nodeRess == null)
return;
- for (QueryResults ress : nodeRess.results().values())
- ress.cancel(true);
+ nodeRess.cancelAll();
}
}, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
@@ -235,12 +238,7 @@ public class GridMapQueryExecutor {
GridH2QueryContext.clear(ctx.localNodeId(), node.id(), qryReqId, MAP);
}
- QueryResults results = nodeRess.results().remove(qryReqId);
-
- if (results == null)
- return;
-
- results.cancel(true);
+ nodeRess.cancelRequest(qryReqId);
}
/**
@@ -427,6 +425,7 @@ public class GridMapQueryExecutor {
onQueryRequest0(node,
req.requestId(),
+ 0,
req.queries(),
cacheIds,
req.topologyVersion(),
@@ -434,7 +433,7 @@ public class GridMapQueryExecutor {
req.partitions(),
null,
req.pageSize(),
- false,
+ OFF,
req.timeout());
}
@@ -442,12 +441,49 @@ public class GridMapQueryExecutor {
* @param node Node.
* @param req Query request.
*/
- private void onQueryRequest(ClusterNode node, GridH2QueryRequest req) {
- Map<UUID,int[]> partsMap = req.partitions();
- int[] parts = partsMap == null ? null : partsMap.get(ctx.localNodeId());
+ private void onQueryRequest(final ClusterNode node, final GridH2QueryRequest req) throws IgniteCheckedException {
+ final Map<UUID,int[]> partsMap = req.partitions();
+ final int[] parts = partsMap == null ? null : partsMap.get(ctx.localNodeId());
+
+ assert req.caches() != null && !req.caches().isEmpty();
+
+ GridCacheContext<?, ?> mainCctx = ctx.cache().context().cacheContext( req.caches().get(0));
+
+ if (mainCctx == null)
+ throw new CacheException("Failed to find cache.");
+
+ final DistributedJoinMode joinMode = distributedJoinMode(
+ req.isFlagSet(GridH2QueryRequest.FLAG_IS_LOCAL),
+ req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS));
+
+ for (int i = 1; i < mainCctx.config().getQueryParallelism(); i++) {
+ final int segment = i;
+
+ ctx.closure().callLocal(
+ new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ onQueryRequest0(node,
+ req.requestId(),
+ segment,
+ req.queries(),
+ req.caches(),
+ req.topologyVersion(),
+ partsMap,
+ parts,
+ req.tables(),
+ req.pageSize(),
+ joinMode,
+ req.timeout());
+
+ return null;
+ }
+ }
+ , QUERY_POOL);
+ }
onQueryRequest0(node,
req.requestId(),
+ 0,
req.queries(),
req.caches(),
req.topologyVersion(),
@@ -455,13 +491,14 @@ public class GridMapQueryExecutor {
parts,
req.tables(),
req.pageSize(),
- req.isFlagSet(GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS),
+ joinMode,
req.timeout());
}
/**
* @param node Node authored request.
* @param reqId Request ID.
+ * @param segmentId index segment ID.
* @param qrys Queries to execute.
* @param cacheIds Caches which will be affected by these queries.
* @param topVer Topology version.
@@ -469,11 +506,12 @@ public class GridMapQueryExecutor {
* @param parts Explicit partitions for current node.
* @param tbls Tables.
* @param pageSize Page size.
- * @param distributedJoins Can we expect distributed joins to be ran.
+ * @param distributedJoinMode Query distributed join mode.
*/
private void onQueryRequest0(
ClusterNode node,
long reqId,
+ int segmentId,
Collection<GridCacheSqlQuery> qrys,
List<Integer> cacheIds,
AffinityTopologyVersion topVer,
@@ -481,7 +519,7 @@ public class GridMapQueryExecutor {
int[] parts,
Collection<String> tbls,
int pageSize,
- boolean distributedJoins,
+ DistributedJoinMode distributedJoinMode,
int timeout
) {
// Prepare to run queries.
@@ -500,7 +538,7 @@ public class GridMapQueryExecutor {
if (topVer != null) {
// Reserve primary for topology version or explicit partitions.
if (!reservePartitions(cacheIds, topVer, parts, reserved)) {
- sendRetry(node, reqId);
+ sendRetry(node, reqId, segmentId);
return;
}
@@ -508,17 +546,18 @@ public class GridMapQueryExecutor {
qr = new QueryResults(reqId, qrys.size(), mainCctx);
- if (nodeRess.results().put(reqId, qr) != null)
+ if (nodeRess.put(reqId, segmentId, qr) != null)
throw new IllegalStateException();
// Prepare query context.
GridH2QueryContext qctx = new GridH2QueryContext(ctx.localNodeId(),
node.id(),
reqId,
+ segmentId,
mainCctx.isReplicated() ? REPLICATED : MAP)
.filter(h2.backupFilter(topVer, parts))
.partitionsMap(partsMap)
- .distributedJoins(distributedJoins)
+ .distributedJoinMode(distributedJoinMode)
.pageSize(pageSize)
.topologyVersion(topVer)
.reservations(reserved);
@@ -542,7 +581,7 @@ public class GridMapQueryExecutor {
Connection conn = h2.connectionForSpace(mainCctx.name());
// Here we enforce join order to have the same behavior on all the nodes.
- h2.setupConnection(conn, distributedJoins, true);
+ h2.setupConnection(conn, distributedJoinMode != OFF, true);
GridH2QueryContext.set(qctx);
@@ -553,13 +592,13 @@ public class GridMapQueryExecutor {
if (nodeRess.cancelled(reqId)) {
GridH2QueryContext.clear(ctx.localNodeId(), node.id(), reqId, qctx.type());
- nodeRess.results().remove(reqId);
+ nodeRess.cancelRequest(reqId);
throw new QueryCancelledException();
}
// Run queries.
- int i = 0;
+ int qryIdx = 0;
boolean evt = ctx.event().isRecordable(EVT_CACHE_QUERY_EXECUTED);
@@ -567,7 +606,7 @@ public class GridMapQueryExecutor {
ResultSet rs = h2.executeSqlQueryWithTimer(mainCctx.name(), conn, qry.query(),
F.asList(qry.parameters()), true,
timeout,
- qr.cancels[i]);
+ qr.cancels[qryIdx]);
if (evt) {
ctx.event().record(new CacheQueryExecutedEvent<>(
@@ -587,24 +626,24 @@ public class GridMapQueryExecutor {
assert rs instanceof JdbcResultSet : rs.getClass();
- qr.addResult(i, qry, node.id(), rs);
+ qr.addResult(qryIdx, qry, node.id(), rs);
if (qr.canceled) {
- qr.result(i).close();
+ qr.result(qryIdx).close();
throw new QueryCancelledException();
}
// Send the first page.
- sendNextPage(nodeRess, node, qr, i, pageSize);
+ sendNextPage(nodeRess, node, qr, qryIdx, segmentId, pageSize);
- i++;
+ qryIdx++;
}
}
finally {
GridH2QueryContext.clearThreadLocal();
- if (!distributedJoins)
+ if (distributedJoinMode == OFF)
qctx.clearContext(false);
if (!F.isEmpty(snapshotedTbls)) {
@@ -615,13 +654,13 @@ public class GridMapQueryExecutor {
}
catch (Throwable e) {
if (qr != null) {
- nodeRess.results().remove(reqId, qr);
+ nodeRess.remove(reqId, segmentId, qr);
qr.cancel(false);
}
if (X.hasCause(e, GridH2RetryException.class))
- sendRetry(node, reqId);
+ sendRetry(node, reqId, segmentId);
else {
U.error(log, "Failed to execute local query.", e);
@@ -681,14 +720,14 @@ public class GridMapQueryExecutor {
return;
}
- QueryResults qr = nodeRess.results().get(req.queryRequestId());
+ QueryResults qr = nodeRess.get(req.queryRequestId(), req.segmentId());
if (qr == null)
sendError(node, req.queryRequestId(), new CacheException("No query result found for request: " + req));
else if (qr.canceled)
sendError(node, req.queryRequestId(), new QueryCancelledException());
else
- sendNextPage(nodeRess, node, qr, req.query(), req.pageSize());
+ sendNextPage(nodeRess, node, qr, req.query(), req.segmentId(), req.pageSize());
}
/**
@@ -696,9 +735,10 @@ public class GridMapQueryExecutor {
* @param node Node.
* @param qr Query results.
* @param qry Query.
+ * @param segmentId Index segment ID.
* @param pageSize Page size.
*/
- private void sendNextPage(NodeResults nodeRess, ClusterNode node, QueryResults qr, int qry,
+ private void sendNextPage(NodeResults nodeRess, ClusterNode node, QueryResults qr, int qry, int segmentId,
int pageSize) {
QueryResult res = qr.result(qry);
@@ -714,14 +754,14 @@ public class GridMapQueryExecutor {
res.close();
if (qr.isAllClosed())
- nodeRess.results().remove(qr.qryReqId, qr);
+ nodeRess.remove(qr.qryReqId, segmentId, qr);
}
try {
boolean loc = node.isLocal();
- GridQueryNextPageResponse msg = new GridQueryNextPageResponse(qr.qryReqId, qry, page,
- page == 0 ? res.rowCnt : -1 ,
+ GridQueryNextPageResponse msg = new GridQueryNextPageResponse(qr.qryReqId, segmentId, qry, page,
+ page == 0 ? res.rowCnt : -1,
res.cols,
loc ? null : toMessages(rows, new ArrayList<Message>(res.cols)),
loc ? rows : null);
@@ -741,12 +781,13 @@ public class GridMapQueryExecutor {
/**
* @param node Node.
* @param reqId Request ID.
+ * @param segmentId Index segment ID.
*/
- private void sendRetry(ClusterNode node, long reqId) {
+ private void sendRetry(ClusterNode node, long reqId, int segmentId) {
try {
boolean loc = node.isLocal();
- GridQueryNextPageResponse msg = new GridQueryNextPageResponse(reqId,
+ GridQueryNextPageResponse msg = new GridQueryNextPageResponse(reqId, segmentId,
/*qry*/0, /*page*/0, /*allRows*/0, /*cols*/1,
loc ? null : Collections.<Message>emptyList(),
loc ? Collections.<Value[]>emptyList() : null);
@@ -780,35 +821,118 @@ public class GridMapQueryExecutor {
*/
private static class NodeResults {
/** */
- private final ConcurrentMap<Long, QueryResults> res = new ConcurrentHashMap8<>();
+ private final ConcurrentMap<RequestKey, QueryResults> res = new ConcurrentHashMap8<>();
/** */
private final GridBoundedConcurrentLinkedHashMap<Long, Boolean> qryHist =
new GridBoundedConcurrentLinkedHashMap<>(1024, 1024, 0.75f, 64, PER_SEGMENT_Q);
/**
- * @return All results.
+ * @param reqId Query Request ID.
+ * @return {@code False} if query was already cancelled.
*/
- ConcurrentMap<Long, QueryResults> results() {
- return res;
+ boolean cancelled(long reqId) {
+ return qryHist.get(reqId) != null;
}
/**
- * @param qryId Query ID.
- * @return {@code False} if query was already cancelled.
+ * @param reqId Query Request ID.
+ * @return {@code True} if cancelled.
*/
- boolean cancelled(long qryId) {
- return qryHist.get(qryId) != null;
+ boolean onCancel(long reqId) {
+ Boolean old = qryHist.putIfAbsent(reqId, Boolean.FALSE);
+
+ return old == null;
}
/**
- * @param qryId Query ID.
- * @return {@code True} if cancelled.
+ * @param reqId Query Request ID.
+ * @param segmentId Index segment ID.
+ * @return query partial results.
*/
- boolean onCancel(long qryId) {
- Boolean old = qryHist.putIfAbsent(qryId, Boolean.FALSE);
+ public QueryResults get(long reqId, int segmentId) {
+ return res.get(new RequestKey(reqId, segmentId));
+ }
- return old == null;
+ /**
+ * Cancel all thread of given request.
+ * @param reqID Request ID.
+ */
+ public void cancelRequest(long reqID) {
+ for (RequestKey key : res.keySet()) {
+ if (key.reqId == reqID) {
+ QueryResults removed = res.remove(key);
+
+ if (removed != null)
+ removed.cancel(true);
+ }
+
+ }
+ }
+
+ /**
+ * @param reqId Query Request ID.
+ * @param segmentId Index segment ID.
+ * @param qr Query Results.
+ * @return {@code True} if removed.
+ */
+ public boolean remove(long reqId, int segmentId, QueryResults qr) {
+ return res.remove(new RequestKey(reqId, segmentId), qr);
+ }
+
+ /**
+ * @param reqId Query Request ID.
+ * @param segmentId Index segment ID.
+ * @param qr Query Results.
+ * @return previous value.
+ */
+ public QueryResults put(long reqId, int segmentId, QueryResults qr) {
+ return res.put(new RequestKey(reqId, segmentId), qr);
+ }
+
+ /**
+ * Cancel all node queries.
+ */
+ public void cancelAll() {
+ for (QueryResults ress : res.values())
+ ress.cancel(true);
+ }
+
+ /**
+ *
+ */
+ private static class RequestKey {
+ /** */
+ private long reqId;
+
+ /** */
+ private int segmentId;
+
+ /** Constructor */
+ RequestKey(long reqId, int segmentId) {
+ this.reqId = reqId;
+ this.segmentId = segmentId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ RequestKey other = (RequestKey)o;
+
+ return reqId == other.reqId && segmentId == other.segmentId;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int result = (int)(reqId ^ (reqId >>> 32));
+ result = 31 * result + segmentId;
+ return result;
+ }
}
}
@@ -836,7 +960,8 @@ public class GridMapQueryExecutor {
* @param qrys Number of queries.
* @param cctx Cache context.
*/
- private QueryResults(long qryReqId, int qrys, GridCacheContext<?,?> cctx) {
+ @SuppressWarnings("unchecked")
+ private QueryResults(long qryReqId, int qrys, GridCacheContext<?, ?> cctx) {
this.qryReqId = qryReqId;
this.cctx = cctx;
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
index c267f4a..45d3c58 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java
@@ -59,7 +59,7 @@ public abstract class GridMergeIndex extends BaseIndex {
private final AtomicInteger expRowsCnt = new AtomicInteger(0);
/** Remaining rows per source node ID. */
- private Map<UUID, Counter> remainingRows;
+ private Map<UUID, Counter[]> remainingRows;
/** */
private final AtomicBoolean lastSubmitted = new AtomicBoolean();
@@ -141,15 +141,22 @@ public abstract class GridMergeIndex extends BaseIndex {
* Set source nodes.
*
* @param nodes Nodes.
+ * @param segmentsCnt Index segments per table.
*/
- public void setSources(Collection<ClusterNode> nodes) {
+ public void setSources(Collection<ClusterNode> nodes, int segmentsCnt) {
assert remainingRows == null;
remainingRows = U.newHashMap(nodes.size());
for (ClusterNode node : nodes) {
- if (remainingRows.put(node.id(), new Counter()) != null)
+ Counter[] counters = new Counter[segmentsCnt];
+
+ for (int i = 0; i < segmentsCnt; i++)
+ counters[i] = new Counter();
+
+ if (remainingRows.put(node.id(), counters) != null)
throw new IllegalStateException("Duplicate node id: " + node.id());
+
}
}
@@ -194,7 +201,7 @@ public abstract class GridMergeIndex extends BaseIndex {
public final void addPage(GridResultPage page) {
int pageRowsCnt = page.rowsInPage();
- Counter cnt = remainingRows.get(page.source());
+ Counter cnt = remainingRows.get(page.source())[page.res.segmentId()];
// RemainingRowsCount should be updated before page adding to avoid race
// in GridMergeIndexUnsorted cursor iterator
@@ -231,13 +238,14 @@ public abstract class GridMergeIndex extends BaseIndex {
// Guarantee that finished state possible only if counter is zero and all pages was added
cnt.state = State.FINISHED;
- for (Counter c : remainingRows.values()) { // Check all the sources.
- if (c.state != State.FINISHED)
- return;
+ for (Counter[] cntrs : remainingRows.values()) { // Check all the sources.
+ for(int i = 0; i < cntrs.length; i++) {
+ if (cntrs[i].state != State.FINISHED)
+ return;
+ }
}
if (lastSubmitted.compareAndSet(false, true)) {
- // Add page-marker that last page was added
addPage0(new GridResultPage(null, page.source(), null) {
@Override public boolean isLast() {
return true;
@@ -256,7 +264,20 @@ public abstract class GridMergeIndex extends BaseIndex {
* @param page Page.
*/
protected void fetchNextPage(GridResultPage page) {
- if (remainingRows.get(page.source()).get() != 0)
+ assert !page.isLast();
+
+ if(page.isFail())
+ page.fetchNextPage(); //rethrow exceptions
+
+ assert page.res != null;
+
+ Counter[] counters = remainingRows.get(page.source());
+
+ int segId = page.res.segmentId();
+
+ Counter counter = counters[segId];
+
+ if (counter.get() != 0)
page.fetchNextPage();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index f54fab6..8837046 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -62,9 +62,9 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartit
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
-import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
import org.apache.ignite.internal.processors.query.GridQueryCancel;
+import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryContext;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
@@ -100,6 +100,7 @@ import org.jsr166.ConcurrentHashMap8;
import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL_FIELDS;
+import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.REDUCE;
/**
@@ -294,6 +295,7 @@ public class GridReduceQueryExecutor {
private void onNextPage(final ClusterNode node, GridQueryNextPageResponse msg) {
final long qryReqId = msg.queryRequestId();
final int qry = msg.query();
+ final int seg = msg.segmentId();
final QueryRun r = runs.get(qryReqId);
@@ -326,7 +328,7 @@ public class GridReduceQueryExecutor {
}
try {
- GridQueryNextPageRequest msg0 = new GridQueryNextPageRequest(qryReqId, qry, pageSize);
+ GridQueryNextPageRequest msg0 = new GridQueryNextPageRequest(qryReqId, qry, seg, pageSize);
if (node.isLocal())
h2.mapQueryExecutor().onMessage(ctx.localNodeId(), msg0);
@@ -514,29 +516,33 @@ public class GridReduceQueryExecutor {
// Explicit partition mapping for unstable topology.
Map<ClusterNode, IntArray> partsMap = null;
- if (isPreloadingActive(cctx, extraSpaces)) {
- if (cctx.isReplicated())
- nodes = replicatedUnstableDataNodes(cctx, extraSpaces);
- else {
- partsMap = partitionedUnstableDataNodes(cctx, extraSpaces);
+ if (qry.isLocal())
+ nodes = Collections.singleton(ctx.discovery().localNode());
+ else {
+ if (isPreloadingActive(cctx, extraSpaces)) {
+ if (cctx.isReplicated())
+ nodes = replicatedUnstableDataNodes(cctx, extraSpaces);
+ else {
+ partsMap = partitionedUnstableDataNodes(cctx, extraSpaces);
- nodes = partsMap == null ? null : partsMap.keySet();
+ nodes = partsMap == null ? null : partsMap.keySet();
+ }
}
- }
- else
- nodes = stableDataNodes(topVer, cctx, extraSpaces);
+ else
+ nodes = stableDataNodes(topVer, cctx, extraSpaces);
- if (nodes == null)
- continue; // Retry.
+ if (nodes == null)
+ continue; // Retry.
- assert !nodes.isEmpty();
+ assert !nodes.isEmpty();
- if (cctx.isReplicated() || qry.explain()) {
- assert qry.explain() || !nodes.contains(ctx.discovery().localNode()) :
- "We must be on a client node.";
+ if (cctx.isReplicated() || qry.explain()) {
+ assert qry.explain() || !nodes.contains(ctx.discovery().localNode()) :
+ "We must be on a client node.";
- // Select random data node to run query on a replicated data or get EXPLAIN PLAN from a single node.
- nodes = Collections.singleton(F.rand(nodes));
+ // Select random data node to run query on a replicated data or get EXPLAIN PLAN from a single node.
+ nodes = Collections.singleton(F.rand(nodes));
+ }
}
final Collection<ClusterNode> finalNodes = nodes;
@@ -545,6 +551,8 @@ public class GridReduceQueryExecutor {
final boolean skipMergeTbl = !qry.explain() && qry.skipMergeTable();
+ final int segmentsPerIndex = cctx.config().getQueryParallelism();
+
for (GridCacheSqlQuery mapQry : qry.mapQueries()) {
GridMergeIndex idx;
@@ -565,12 +573,12 @@ public class GridReduceQueryExecutor {
else
idx = GridMergeIndexUnsorted.createDummy(ctx);
- idx.setSources(nodes);
+ idx.setSources(nodes, segmentsPerIndex);
r.idxs.add(idx);
}
- r.latch = new CountDownLatch(r.idxs.size() * nodes.size());
+ r.latch = new CountDownLatch(r.idxs.size() * nodes.size() * segmentsPerIndex);
runs.put(qryReqId, r);
@@ -626,14 +634,13 @@ public class GridReduceQueryExecutor {
.tables(distributedJoins ? qry.tables() : null)
.partitions(convert(partsMap))
.queries(mapQrys)
- .flags(distributedJoins ? GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS : 0)
+ .flags((qry.isLocal() ? GridH2QueryRequest.FLAG_IS_LOCAL : 0) |
+ (distributedJoins ? GridH2QueryRequest.FLAG_DISTRIBUTED_JOINS : 0))
.timeout(timeoutMillis),
oldStyle && partsMap != null ? new ExplicitPartitionsSpecializer(partsMap) : null,
- distributedJoins)
- ) {
- awaitAllReplies(r, nodes);
+ false)) {
- cancel.checkCancelled();
+ awaitAllReplies(r, nodes, cancel);
Object state = r.state.get();
@@ -696,7 +703,7 @@ public class GridReduceQueryExecutor {
h2.setupConnection(r.conn, false, enforceJoinOrder);
GridH2QueryContext.set(new GridH2QueryContext(locNodeId, locNodeId, qryReqId, REDUCE)
- .pageSize(r.pageSize).distributedJoins(false));
+ .pageSize(r.pageSize).distributedJoinMode(OFF));
try {
if (qry.explain())
@@ -817,11 +824,15 @@ public class GridReduceQueryExecutor {
/**
* @param r Query run.
* @param nodes Nodes to check periodically if they alive.
+ * @param cancel Query cancel.
* @throws IgniteInterruptedCheckedException If interrupted.
*/
- private void awaitAllReplies(QueryRun r, Collection<ClusterNode> nodes)
- throws IgniteInterruptedCheckedException {
+ private void awaitAllReplies(QueryRun r, Collection<ClusterNode> nodes, GridQueryCancel cancel)
+ throws IgniteInterruptedCheckedException, QueryCancelledException {
while (!U.await(r.latch, 500, TimeUnit.MILLISECONDS)) {
+
+ cancel.checkCancelled();
+
for (ClusterNode node : nodes) {
if (!ctx.discovery().alive(node)) {
handleNodeLeft(r, node.id());
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
index e49c48f..b2548cc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
@@ -38,6 +38,12 @@ public class GridH2IndexRangeRequest implements Message {
private long qryId;
/** */
+ private int originSegmentId;
+
+ /** */
+ private int segmentId;
+
+ /** */
private int batchLookupId;
/** */
@@ -87,6 +93,34 @@ public class GridH2IndexRangeRequest implements Message {
}
/**
+ * @param segmentId Index segment ID.
+ */
+ public void segment(int segmentId) {
+ this.segmentId = segmentId;
+ }
+
+ /**
+ * @return Index segment ID.
+ */
+ public int segment() {
+ return segmentId;
+ }
+
+ /**
+ * @return Origin index segment ID.
+ */
+ public int originSegmentId() {
+ return originSegmentId;
+ }
+
+ /**
+ * @param segmentId Origin index segment ID.
+ */
+ public void originSegmentId(int segmentId) {
+ this.originSegmentId = segmentId;
+ }
+
+ /**
* @param batchLookupId Batch lookup ID.
*/
public void batchLookupId(int batchLookupId) {
@@ -136,6 +170,15 @@ public class GridH2IndexRangeRequest implements Message {
writer.incrementState();
+ case 4:
+ if (!writer.writeInt("segmentId", segmentId))
+ return false;
+
+ case 5:
+ if (!writer.writeInt("originSegId", originSegmentId))
+ return false;
+
+ writer.incrementState();
}
return true;
@@ -181,6 +224,21 @@ public class GridH2IndexRangeRequest implements Message {
reader.incrementState();
+ case 4:
+ segmentId = reader.readInt("segmentId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 5:
+ originSegmentId = reader.readInt("originSegId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
}
return reader.afterMessageRead(GridH2IndexRangeRequest.class);
@@ -193,7 +251,7 @@ public class GridH2IndexRangeRequest implements Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 4;
+ return 6;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java
index c6414bd..4d3db12 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java
@@ -47,6 +47,12 @@ public class GridH2IndexRangeResponse implements Message {
private long qryId;
/** */
+ private int segmentId;
+
+ /** */
+ private int originSegmentId;
+
+ /** */
private int batchLookupId;
/** */
@@ -130,6 +136,34 @@ public class GridH2IndexRangeResponse implements Message {
}
/**
+ * @param segmentId Index segment ID.
+ */
+ public void segment(int segmentId) {
+ this.segmentId = segmentId;
+ }
+
+ /**
+ * @return Index segment ID.
+ */
+ public int segment() {
+ return segmentId;
+ }
+
+ /**
+ * @return Origin index segment ID.
+ */
+ public int originSegmentId() {
+ return originSegmentId;
+ }
+
+ /**
+ * @param segmentId Origin index segment ID.
+ */
+ public void originSegmentId(int segmentId) {
+ this.originSegmentId = segmentId;
+ }
+
+ /**
* @param batchLookupId Batch lookup ID.
*/
public void batchLookupId(int batchLookupId) {
@@ -191,6 +225,17 @@ public class GridH2IndexRangeResponse implements Message {
writer.incrementState();
+ case 6:
+ if (!writer.writeInt("originSegId", originSegmentId))
+ return false;
+
+ writer.incrementState();
+
+ case 7:
+ if (!writer.writeInt("segmentId", segmentId))
+ return false;
+
+ writer.incrementState();
}
return true;
@@ -252,6 +297,21 @@ public class GridH2IndexRangeResponse implements Message {
reader.incrementState();
+ case 6:
+ originSegmentId = reader.readInt("originSegId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 7:
+ segmentId = reader.readInt("segmentId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
}
return reader.afterMessageRead(GridH2IndexRangeResponse.class);
@@ -264,7 +324,7 @@ public class GridH2IndexRangeResponse implements Message {
/** {@inheritDoc} */
@Override public byte fieldsCount() {
- return 6;
+ return 8;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
index 884173f..ec49aff 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
@@ -50,6 +50,11 @@ public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
*/
public static int FLAG_DISTRIBUTED_JOINS = 1;
+ /**
+ * Restrict distributed joins range-requests to local index segments. Range requests to other nodes will not be sent.
+ */
+ public static int FLAG_IS_LOCAL = 2;
+
/** */
private long reqId;
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
new file mode 100644
index 0000000..f8c9dd5
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSegmentedIndexSelfTest.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheKeyConfiguration;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for correct distributed queries with index consisted of many segments.
+ */
+public class IgniteSqlSegmentedIndexSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static int QRY_PARALLELISM_LVL = 97;
+
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheKeyConfiguration keyCfg = new CacheKeyConfiguration("MyCache", "affKey");
+
+ cfg.setCacheKeyConfiguration(keyCfg);
+
+ cfg.setPeerClassLoadingEnabled(false);
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+ disco.setIpFinder(ipFinder);
+
+ cfg.setDiscoverySpi(disco);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+ }
+
+ /**
+ * @param name Cache name.
+ * @param partitioned Partition or replicated cache.
+ * @param idxTypes Indexed types.
+ * @return Cache configuration.
+ */
+ private static <K, V> CacheConfiguration<K, V> cacheConfig(String name, boolean partitioned, Class<?>... idxTypes) {
+ return new CacheConfiguration<K, V>()
+ .setName(name)
+ .setCacheMode(partitioned ? CacheMode.PARTITIONED : CacheMode.REPLICATED)
+ .setQueryParallelism(partitioned ? QRY_PARALLELISM_LVL : 1)
+ .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+ .setIndexedTypes(idxTypes);
+ }
+
+ /**
+ * Run tests on single-node grid
+ * @throws Exception If failed.
+ */
+ public void testSingleNodeIndexSegmentation() throws Exception {
+ startGridsMultiThreaded(1, true);
+
+ ignite(0).createCache(cacheConfig("pers", true, Integer.class, Person.class));
+ ignite(0).createCache(cacheConfig("org", true, Integer.class, Organization.class));
+
+ fillCache();
+
+ checkDistributedQueryWithSegmentedIndex();
+
+ checkLocalQueryWithSegmentedIndex();
+ }
+
+ /**
+ * Run tests on multi-node grid
+ * @throws Exception If failed.
+ */
+ public void testMultiNodeIndexSegmentation() throws Exception {
+ startGridsMultiThreaded(4, true);
+
+ ignite(0).createCache(cacheConfig("pers", true, Integer.class, Person.class));
+ ignite(0).createCache(cacheConfig("org", true, Integer.class, Organization.class));
+
+ fillCache();
+
+ checkDistributedQueryWithSegmentedIndex();
+
+ checkLocalQueryWithSegmentedIndex();
+ }
+
+ /**
+ * Run tests on multi-node grid
+ * @throws Exception If failed.
+ */
+ public void testMultiNodeSegmentedPartitionedWithReplicated() throws Exception {
+ startGridsMultiThreaded(4, true);
+
+ ignite(0).createCache(cacheConfig("pers", true, Integer.class, Person.class));
+ ignite(0).createCache(cacheConfig("org", false, Integer.class, Organization.class));
+
+ fillCache();
+
+ checkDistributedQueryWithSegmentedIndex();
+
+ checkLocalQueryWithSegmentedIndex();
+ }
+
+ /**
+ * Check distributed joins.
+ * @throws Exception If failed.
+ */
+ public void checkDistributedQueryWithSegmentedIndex() throws Exception {
+ IgniteCache<Integer, Person> c1 = ignite(0).cache("pers");
+
+ int expectedPersons = 0;
+
+ for (Cache.Entry<Integer, Person> e : c1) {
+ final Integer orgId = e.getValue().orgId;
+
+ if (10 <= orgId && orgId < 500)
+ expectedPersons++;
+ }
+
+ String select0 = "select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
+
+ List<List<?>> result = c1.query(new SqlFieldsQuery(select0).setDistributedJoins(true)).getAll();
+
+ assertEquals(expectedPersons, result.size());
+ }
+
+ /**
+ * Test local query.
+ * @throws Exception If failed.
+ */
+ public void checkLocalQueryWithSegmentedIndex() throws Exception {
+ IgniteCache<Integer, Person> c1 = ignite(0).cache("pers");
+ IgniteCache<Integer, Organization> c2 = ignite(0).cache("org");
+
+ Set<Integer> localOrgIds = new HashSet<>();
+
+ for(Cache.Entry<Integer, Organization> e : c2.localEntries())
+ localOrgIds.add(e.getKey());
+
+ int expectedPersons = 0;
+
+ for (Cache.Entry<Integer, Person> e : c1.localEntries()) {
+ final Integer orgId = e.getValue().orgId;
+
+ if (localOrgIds.contains(orgId))
+ expectedPersons++;
+ }
+
+ String select0 = "select o.name n1, p.name n2 from \"pers\".Person p, \"org\".Organization o where p.orgId = o._key";
+
+ List<List<?>> result = c1.query(new SqlFieldsQuery(select0).setLocal(true)).getAll();
+
+ assertEquals(expectedPersons, result.size());
+ }
+
+ /** */
+ private void fillCache() {
+ IgniteCache<Object, Object> c1 = ignite(0).cache("pers");
+
+ IgniteCache<Object, Object> c2 = ignite(0).cache("org");
+
+ final int orgCount = 500;
+
+ for (int i = 0; i < orgCount; i++)
+ c2.put(i, new Organization("org-" + i));
+
+ final Random random = new Random();
+
+ for (int i = 0; i < 1000; i++) {
+ int orgID = 10 + random.nextInt(orgCount + 10);
+
+ c1.put(i, new Person(orgID, "pers-" + i));
+ }
+ }
+
+ /**
+ *
+ */
+ private static class Person implements Serializable {
+ /** */
+ @QuerySqlField(index = true)
+ Integer orgId;
+
+ /** */
+ @QuerySqlField
+ String name;
+
+ /**
+ *
+ */
+ public Person() {
+ // No-op.
+ }
+
+ /**
+ * @param orgId Organization ID.
+ * @param name Name.
+ */
+ public Person(int orgId, String name) {
+ this.orgId = orgId;
+ this.name = name;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class Organization implements Serializable {
+ /** */
+ @QuerySqlField
+ String name;
+
+ /**
+ *
+ */
+ public Organization() {
+ // No-op.
+ }
+
+ /**
+ * @param name Organization name.
+ */
+ public Organization(String name) {
+ this.name = name;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
index 06afe7c..4ae2f91 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSplitterSelfTest.java
@@ -33,9 +33,10 @@ import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheKeyConfiguration;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
+import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.cluster.ClusterNode;
@@ -701,6 +702,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
ignite(0).destroyCache(cache.getName());
}
}
+
/**
* @throws Exception If failed.
*/
@@ -750,6 +752,127 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ public void testIndexSegmentation() throws Exception {
+ CacheConfiguration ccfg1 = cacheConfig("pers", true,
+ Integer.class, Person2.class).setQueryParallelism(4);
+ CacheConfiguration ccfg2 = cacheConfig("org", true,
+ Integer.class, Organization.class).setQueryParallelism(4);
+
+ IgniteCache<Object, Object> c1 = ignite(0).getOrCreateCache(ccfg1);
+ IgniteCache<Object, Object> c2 = ignite(0).getOrCreateCache(ccfg2);
+
+ try {
+ c2.put(1, new Organization("o1"));
+ c2.put(2, new Organization("o2"));
+ c1.put(3, new Person2(1, "p1"));
+ c1.put(4, new Person2(2, "p2"));
+ c1.put(5, new Person2(3, "p3"));
+
+ String select0 = "select o.name n1, p.name n2 from \"pers\".Person2 p, \"org\".Organization o where p.orgId = o._key and o._key=1";
+
+ checkQueryPlan(c1, true, 1, new SqlFieldsQuery(select0));
+
+ checkQueryPlan(c1, true, 1, new SqlFieldsQuery(select0).setLocal(true));
+ }
+ finally {
+ c1.destroy();
+ c2.destroy();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReplicationCacheIndexSegmentationFailure() throws Exception {
+ GridTestUtils.assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ CacheConfiguration ccfg = cacheConfig("org", false,
+ Integer.class, Organization.class).setQueryParallelism(4);
+
+ IgniteCache<Object, Object> c = ignite(0).createCache(ccfg);
+
+ return null;
+ }
+ }, CacheException.class, "Cache index segmentation is supported for PARTITIONED mode only.");
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testIndexSegmentationPartitionedReplicated() throws Exception {
+ CacheConfiguration ccfg1 = cacheConfig("pers", true,
+ Integer.class, Person2.class).setQueryParallelism(4);
+ CacheConfiguration ccfg2 = cacheConfig("org", false,
+ Integer.class, Organization.class);
+
+ final IgniteCache<Object, Object> c1 = ignite(0).getOrCreateCache(ccfg1);
+ final IgniteCache<Object, Object> c2 = ignite(0).getOrCreateCache(ccfg2);
+
+ try {
+ c2.put(1, new Organization("o1"));
+ c2.put(2, new Organization("o2"));
+ c1.put(3, new Person2(1, "p1"));
+ c1.put(4, new Person2(2, "p2"));
+ c1.put(5, new Person2(3, "p3"));
+
+ String select0 = "select o.name n1, p.name n2 from \"pers\".Person2 p, \"org\".Organization o where p.orgId = o._key";
+
+ final SqlFieldsQuery qry = new SqlFieldsQuery(select0);
+
+ qry.setDistributedJoins(true);
+
+ List<List<?>> results = c1.query(qry).getAll();
+
+ assertEquals(2, results.size());
+ }
+ finally {
+ c1.destroy();
+ c2.destroy();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testIndexWithDifferentSegmentationLevelsFailure() throws Exception {
+ CacheConfiguration ccfg1 = cacheConfig("pers", true,
+ Integer.class, Person2.class).setQueryParallelism(4);
+ CacheConfiguration ccfg2 = cacheConfig("org", true,
+ Integer.class, Organization.class).setQueryParallelism(3);
+
+ final IgniteCache<Object, Object> c1 = ignite(0).getOrCreateCache(ccfg1);
+ final IgniteCache<Object, Object> c2 = ignite(0).getOrCreateCache(ccfg2);
+
+ try {
+ c2.put(1, new Organization("o1"));
+ c2.put(2, new Organization("o2"));
+ c1.put(3, new Person2(1, "p1"));
+ c1.put(4, new Person2(2, "p2"));
+ c1.put(5, new Person2(3, "p3"));
+
+ String select0 = "select o.name n1, p.name n2 from \"pers\".Person2 p, \"org\".Organization o where p.orgId = o._key and o._key=1";
+
+ final SqlFieldsQuery qry = new SqlFieldsQuery(select0);
+
+ qry.setDistributedJoins(true);
+
+ GridTestUtils.assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ c1.query(qry);
+
+ return null;
+ }
+ }, CacheException.class, "Using indexes with different parallelism levels in same query is forbidden.");
+ }
+ finally {
+ c1.destroy();
+ c2.destroy();
+ }
+ }
+
+ /**
* @param cache Cache.
* @param sql SQL.
* @param enforceJoinOrder Enforce join order flag.
@@ -789,26 +912,26 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
false,
0,
select +
- "from " + cache1 + "," + cache2 + " "+ where);
+ "from " + cache1 + "," + cache2 + " " + where);
checkQueryPlan(cache,
false,
0,
select +
- "from " + cache2 + "," + cache1 + " "+ where);
+ "from " + cache2 + "," + cache1 + " " + where);
if (testEnforceJoinOrder) {
checkQueryPlan(cache,
true,
0,
select +
- "from " + cache1 + "," + cache2 + " "+ where);
+ "from " + cache1 + "," + cache2 + " " + where);
checkQueryPlan(cache,
true,
0,
select +
- "from " + cache2 + "," + cache1 + " "+ where);
+ "from " + cache2 + "," + cache1 + " " + where);
}
}
@@ -823,7 +946,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
boolean enforceJoinOrder,
int expBatchedJoins,
String sql,
- String...expText) {
+ String... expText) {
checkQueryPlan(cache,
enforceJoinOrder,
expBatchedJoins,
@@ -848,7 +971,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
boolean enforceJoinOrder,
int expBatchedJoins,
SqlFieldsQuery qry,
- String...expText) {
+ String... expText) {
qry.setEnforceJoinOrder(enforceJoinOrder);
qry.setDistributedJoins(true);
@@ -984,7 +1107,7 @@ public class IgniteSqlSplitterSelfTest extends GridCommonAbstractTest {
* @param args Arguments.
* @return Column as list.
*/
- private static <X> List<X> columnQuery(IgniteCache<?,?> c, String qry, Object... args) {
+ private static <X> List<X> columnQuery(IgniteCache<?, ?> c, String qry, Object... args) {
return column(0, c.query(new SqlFieldsQuery(qry).setArgs(args)).getAll());
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
index 52084c7..d6a5fb1 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexingSpiAbstractSelfTest.java
@@ -230,16 +230,16 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
assertEquals(0, spi.size(typeAB.space(), typeAB));
assertEquals(0, spi.size(typeBA.space(), typeBA));
- assertFalse(spi.queryLocalSql(typeAA.space(), "select * from A.A", null, Collections.emptySet(), typeAA, null).hasNext());
- assertFalse(spi.queryLocalSql(typeAB.space(), "select * from A.B", null, Collections.emptySet(), typeAB, null).hasNext());
- assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A", null, Collections.emptySet(), typeBA, null).hasNext());
+ assertFalse(spi.queryLocalSql(typeAA.space(), "select * from A.A", null, Collections.emptySet(), typeAA.name(), null, null).hasNext());
+ assertFalse(spi.queryLocalSql(typeAB.space(), "select * from A.B", null, Collections.emptySet(), typeAB.name(), null, null).hasNext());
+ assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A", null, Collections.emptySet(), typeBA.name(), null, null).hasNext());
assertFalse(spi.queryLocalSql(typeBA.space(), "select * from B.A, A.B, A.A", null,
- Collections.emptySet(), typeBA, null).hasNext());
+ Collections.emptySet(), typeBA.name(), null, null).hasNext());
try {
spi.queryLocalSql(typeBA.space(), "select aa.*, ab.*, ba.* from A.A aa, A.B ab, B.A ba", null,
- Collections.emptySet(), typeBA, null).hasNext();
+ Collections.emptySet(), typeBA.name(), null, null).hasNext();
fail("Enumerations of aliases in select block must be prohibited");
}
@@ -248,10 +248,10 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
}
assertFalse(spi.queryLocalSql(typeAB.space(), "select ab.* from A.B ab", null,
- Collections.emptySet(), typeAB, null).hasNext());
+ Collections.emptySet(), typeAB.name(), null, null).hasNext());
assertFalse(spi.queryLocalSql(typeBA.space(), "select ba.* from B.A as ba", null,
- Collections.emptySet(), typeBA, null).hasNext());
+ Collections.emptySet(), typeBA.name(), null, null).hasNext());
// Nothing to remove.
spi.remove("A", key(1), aa(1, "", 10));
@@ -305,7 +305,7 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
// Query data.
Iterator<IgniteBiTuple<Integer, Map<String, Object>>> res =
- spi.queryLocalSql(typeAA.space(), "from a order by age", null, Collections.emptySet(), typeAA, null);
+ spi.queryLocalSql(typeAA.space(), "from a order by age", null, Collections.emptySet(), typeAA.name(), null, null);
assertTrue(res.hasNext());
assertEquals(aa(3, "Borya", 18).value(null, false), value(res.next()));
@@ -314,7 +314,7 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
assertFalse(res.hasNext());
res = spi.queryLocalSql(typeAA.space(), "select aa.* from a aa order by aa.age", null,
- Collections.emptySet(), typeAA, null);
+ Collections.emptySet(), typeAA.name(), null, null);
assertTrue(res.hasNext());
assertEquals(aa(3, "Borya", 18).value(null, false), value(res.next()));
@@ -322,7 +322,7 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
assertEquals(aa(2, "Valera", 19).value(null, false), value(res.next()));
assertFalse(res.hasNext());
- res = spi.queryLocalSql(typeAB.space(), "from b order by name", null, Collections.emptySet(), typeAB, null);
+ res = spi.queryLocalSql(typeAB.space(), "from b order by name", null, Collections.emptySet(), typeAB.name(), null, null);
assertTrue(res.hasNext());
assertEquals(ab(1, "Vasya", 20, "Some text about Vasya goes here.").value(null, false), value(res.next()));
@@ -331,7 +331,7 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
assertFalse(res.hasNext());
res = spi.queryLocalSql(typeAB.space(), "select bb.* from b as bb order by bb.name", null,
- Collections.emptySet(), typeAB, null);
+ Collections.emptySet(), typeAB.name(), null, null);
assertTrue(res.hasNext());
assertEquals(ab(1, "Vasya", 20, "Some text about Vasya goes here.").value(null, false), value(res.next()));
@@ -340,7 +340,7 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
assertFalse(res.hasNext());
- res = spi.queryLocalSql(typeBA.space(), "from a", null, Collections.emptySet(), typeBA, null);
+ res = spi.queryLocalSql(typeBA.space(), "from a", null, Collections.emptySet(), typeBA.name(), null, null);
assertTrue(res.hasNext());
assertEquals(ba(2, "Kolya", 25, true).value(null, false), value(res.next()));
@@ -725,4 +725,4 @@ public abstract class GridIndexingSpiAbstractSelfTest extends GridCommonAbstract
throw new UnsupportedOperationException();
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/FetchingQueryCursorStressTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/FetchingQueryCursorStressTest.java b/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/FetchingQueryCursorStressTest.java
new file mode 100644
index 0000000..d1d80f2
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/loadtests/h2indexing/FetchingQueryCursorStressTest.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.loadtests.h2indexing;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
+
+/**
+ * SQL query stress test.
+ */
+public class FetchingQueryCursorStressTest {
+ /** Node count. */
+ private static final int NODE_CNT = 4; // Switch to 4 to see better throughput.
+
+ /** Number of entries. */
+ private static final int ENTRIES_CNT = 10_000;
+
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** Thread count. */
+ private static final int THREAD_CNT = 16;
+
+ /** Execution counter. */
+ private static final AtomicLong CNT = new AtomicLong();
+
+ /** Verbose mode. */
+ private static final boolean VERBOSE = false;
+
+ /** */
+ private static final long TIMEOUT = TimeUnit.SECONDS.toMillis(30);
+
+ public static final AtomicReference<Exception> error = new AtomicReference<>();
+
+ /**
+ * Entry point.
+ */
+ public static void main(String[] args) throws Exception {
+ List<Thread> threads = new ArrayList<>(THREAD_CNT + 1);
+
+ try (Ignite ignite = start()) {
+
+ IgniteCache<Integer, Person> cache = ignite.cache(CACHE_NAME);
+
+ loadData(ignite, cache);
+
+ System.out.println("Loaded data: " + cache.size());
+
+ for (int i = 0; i < THREAD_CNT; i++)
+ threads.add(startDaemon("qry-exec-" + i, new QueryExecutor(cache, "Select * from Person")));
+
+ threads.add(startDaemon("printer", new ThroughputPrinter()));
+
+ Thread.sleep(TIMEOUT);
+
+ for (Thread t : threads)
+ t.join();
+
+ if(error.get()!=null)
+ throw error.get();
+ }
+ finally {
+ Ignition.stopAll(false);
+ }
+ }
+
+ /**
+ * Start daemon thread.
+ *
+ * @param name Name.
+ * @param r Runnable.
+ */
+ private static Thread startDaemon(String name, Runnable r) {
+ Thread t = new Thread(r);
+
+ t.setName(name);
+ t.setDaemon(true);
+
+ t.start();
+
+ return t;
+ }
+
+ /**
+ * Load data into Ignite.
+ *
+ * @param ignite Ignite.
+ * @param cache Cache.
+ */
+ private static void loadData(Ignite ignite, IgniteCache<Integer, Person> cache) throws Exception {
+ try (IgniteDataStreamer<Object, Object> str = ignite.dataStreamer(cache.getName())) {
+
+ for (int id = 0; id < ENTRIES_CNT; id++)
+ str.addData(id, new Person(id, "John" + id, "Doe"));
+ }
+ }
+
+ /**
+ * Start topology.
+ *
+ * @return Client node.
+ */
+ private static Ignite start() {
+ int i = 0;
+
+ for (; i < NODE_CNT; i++)
+ Ignition.start(config(i, false));
+
+ return Ignition.start(config(i, true));
+ }
+
+ /**
+ * Create configuration.
+ *
+ * @param idx Index.
+ * @param client Client flag.
+ * @return Configuration.
+ */
+ @SuppressWarnings("unchecked")
+ private static IgniteConfiguration config(int idx, boolean client) {
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("grid-" + idx);
+ cfg.setClientMode(client);
+
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setName(CACHE_NAME);
+ ccfg.setIndexedTypes(Integer.class, Person.class);
+ cfg.setMarshaller(new OptimizedMarshaller());
+
+ cfg.setCacheConfiguration(ccfg);
+
+ cfg.setLocalHost("127.0.0.1");
+
+ return cfg;
+ }
+
+ /**
+ *
+ */
+ private static class Person implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @QuerySqlField
+ private int id;
+
+ /** */
+ @QuerySqlField
+ private String firstName;
+
+ /** */
+ @QuerySqlField
+ private String lastName;
+
+ public Person(int id, String firstName, String lastName) {
+ this.id = id;
+ this.firstName = firstName;
+ this.lastName = lastName;
+ }
+ }
+
+ /**
+ * Query runner.
+ */
+ private static class QueryExecutor implements Runnable {
+ /** Cache. */
+ private final IgniteCache<Integer, Person> cache;
+
+ /** */
+ private final String query;
+
+ /**
+ * Constructor.
+ *
+ * @param cache Cache.
+ */
+ public QueryExecutor(IgniteCache<Integer, Person> cache, String query) {
+ this.cache = cache;
+ this.query = query;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("InfiniteLoopStatement")
+ @Override public void run() {
+ System.out.println("Executor started: " + Thread.currentThread().getName());
+
+ try {
+ while (error.get()==null && !Thread.currentThread().isInterrupted()) {
+ long start = System.nanoTime();
+
+ SqlFieldsQuery qry = new SqlFieldsQuery(query);
+
+// qry.setArgs((Object[]) argumentForQuery());
+
+ Set<Integer> extIds = new HashSet<>();
+
+ for (List<?> next : cache.query(qry))
+ extIds.add((Integer)next.get(0));
+
+ long dur = (System.nanoTime() - start) / 1_000_000;
+
+ CNT.incrementAndGet();
+
+ if (VERBOSE)
+ System.out.println("[extIds=" + extIds.size() + ", dur=" + dur + ']');
+ }
+ }
+ catch (CacheException ex){
+ error.compareAndSet(null, ex);
+ }
+ }
+ }
+
+ /**
+ * Throughput printer.
+ */
+ private static class ThroughputPrinter implements Runnable {
+ /** {@inheritDoc} */
+ @SuppressWarnings("InfiniteLoopStatement")
+ @Override public void run() {
+ while (error.get()==null) {
+ long before = CNT.get();
+ long beforeTime = System.currentTimeMillis();
+
+ try {
+ Thread.sleep(2000L);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ return;
+ }
+
+ long after = CNT.get();
+ long afterTime = System.currentTimeMillis();
+
+ double res = 1000 * ((double)(after - before)) / (afterTime - beforeTime);
+
+ System.out.println((long)res + " ops/sec");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/64ba13b0/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 386b8fd..b417b0a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -98,6 +98,7 @@ import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTes
import org.apache.ignite.internal.processors.cache.query.IndexingSpiQueryTxSelfTest;
import org.apache.ignite.internal.processors.query.IgniteSqlEntryCacheModeAgnosticTest;
import org.apache.ignite.internal.processors.query.IgniteSqlSchemaIndexingTest;
+import org.apache.ignite.internal.processors.query.IgniteSqlSegmentedIndexSelfTest;
import org.apache.ignite.internal.processors.query.IgniteSqlSplitterSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexRebuildTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexingInMemSelfTest;
@@ -134,6 +135,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
// Queries tests.
suite.addTestSuite(IgniteSqlSplitterSelfTest.class);
+ suite.addTestSuite(IgniteSqlSegmentedIndexSelfTest.class);
suite.addTestSuite(IgniteSqlSchemaIndexingTest.class);
suite.addTestSuite(GridCacheQueryIndexDisabledSelfTest.class);
suite.addTestSuite(IgniteCacheQueryLoadSelfTest.class);