You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2017/03/01 14:33:11 UTC
[28/50] [abbrv] ignite git commit: ignite-3860 - fixes2
ignite-3860 - fixes2
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bf337bc5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bf337bc5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bf337bc5
Branch: refs/heads/master
Commit: bf337bc53bca0a1a0ab6b8b5d4f19e509287ebe9
Parents: 207a669
Author: Sergi Vladykin <se...@gmail.com>
Authored: Tue Feb 21 18:53:16 2017 +0300
Committer: Sergi Vladykin <se...@gmail.com>
Committed: Tue Feb 21 18:53:16 2017 +0300
----------------------------------------------------------------------
.../query/h2/sql/GridSqlQuerySplitter.java | 52 ++++++++++++------
.../query/h2/sql/GridSqlStatement.java | 5 ++
.../query/h2/sql/GridSqlSubquery.java | 2 +-
.../h2/sql/AbstractH2CompareQueryTest.java | 21 +++-----
.../query/h2/sql/H2CompareBigQueryTest.java | 56 +++++++++++++-------
.../processors/query/h2/sql/bigQuery.sql | 2 +-
6 files changed, 86 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 165b6b8..a215f20 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -21,6 +21,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -35,7 +36,9 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.h2.command.Prepared;
import org.h2.command.dml.Query;
import org.h2.command.dml.SelectUnion;
@@ -407,15 +410,16 @@ public class GridSqlQuerySplitter {
* @param qrym Query model.
*/
private void injectSortingFirstJoin(QueryModel qrym) {
- // Must always be generated unique aliases here.
- GridSqlAlias leftTbl = qrym.get(0).ast();
- GridSqlAlias rightTbl = qrym.get(1).ast();
-
GridSqlJoin join = findJoin(qrym, 0);
// We are at the beginning, thus left and right AST must be children of the same join AST.
- assert join.leftTable() == leftTbl: join.getSQL();
- assert join.rightTable() == rightTbl: join.getSQL();
+ // join2
+ // / \
+ // join1 \
+ // / \ \
+ // T0 T1 T2
+ GridSqlAlias leftTbl = (GridSqlAlias)join.leftTable();
+ GridSqlAlias rightTbl = (GridSqlAlias)join.rightTable();
// Collect all AND conditions.
List<AndCondition> andConditions = new ArrayList<>();
@@ -461,10 +465,15 @@ public class GridSqlQuerySplitter {
* @param orderByCols Columns for ORDER BY.
*/
private void injectOrderBy(GridSqlAlias subQryAlias, List<GridSqlColumn> orderByCols) {
+ if (orderByCols.isEmpty())
+ return;
+
// Structure: alias -> subquery -> query
- GridSqlQuery qry = subQryAlias.child().child();
+ GridSqlQuery qry = GridSqlAlias.<GridSqlSubquery>unwrap(subQryAlias).subquery();
GridSqlSelect select = leftmostSelect(qry); // The leftmost SELECT in UNION defines column names.
+ BitSet set = new BitSet();
+
for (int i = 0; i < orderByCols.size(); i++) {
GridSqlColumn col = orderByCols.get(i);
@@ -480,12 +489,21 @@ public class GridSqlQuerySplitter {
else if (expr instanceof GridSqlColumn)
colName = ((GridSqlColumn)expr).columnName();
else
- throw new IllegalStateException(); // It must be impossible to join by this column then.
+ // It must be impossible to join by this column then, because the expression has no name.
+ throw new IllegalStateException();
if (colName.equals(col.columnName()))
break; // Found the needed column index.
+
+ colIdx++;
}
+ // Avoid duplicates.
+ if (set.get(colIdx))
+ continue;
+
+ set.set(colIdx, true);
+
// Add sort column to the query.
qry.addSort(new GridSqlSortColumn(colIdx, true, false, false));
}
@@ -591,8 +609,6 @@ public class GridSqlQuerySplitter {
for (int i = begin; i <= end; i++) {
QueryModel child = qrym.get(i);
- assert !child.needSplit && !child.needSplitChild;
-
wrapQrym.add(child);
}
@@ -620,7 +636,7 @@ public class GridSqlQuerySplitter {
int end,
GridSqlAlias wrapAlias
) {
- GridSqlSelect wrapSelect = wrapAlias.child();
+ GridSqlSelect wrapSelect = GridSqlAlias.<GridSqlSubquery>unwrap(wrapAlias).subquery();
final int last = qrym.size() - 1;
@@ -887,7 +903,7 @@ public class GridSqlQuerySplitter {
if (select.where() == null)
return;
- GridSqlSelect wrapSelect = wrapAlias.child();
+ GridSqlSelect wrapSelect = GridSqlAlias.<GridSqlSubquery>unwrap(wrapAlias).subquery();
List<AndCondition> andConditions = new ArrayList<>();
@@ -932,7 +948,7 @@ public class GridSqlQuerySplitter {
}
else {
for (int i = 0; i < ast.size(); i++) {
- if (!isAllRelatedToTables(tblAliases, ast))
+ if (!isAllRelatedToTables(tblAliases, ast.child(i)))
return false;
}
}
@@ -995,8 +1011,6 @@ public class GridSqlQuerySplitter {
for (int i = qrym.size() - 1; i > idx; i--)
join = (GridSqlJoin)join.leftTable();
- assert join.rightTable() == qrym.get(idx).ast();
-
return join;
}
@@ -1956,6 +1970,7 @@ public class GridSqlQuerySplitter {
*/
private static final class QueryModel extends ArrayList<QueryModel> {
/** */
+ @GridToStringInclude
final Type type;
/** */
@@ -1968,9 +1983,11 @@ public class GridSqlQuerySplitter {
int childIdx;
/** If it is a SELECT and we need to split it. Makes sense only for type SELECT. */
+ @GridToStringInclude
boolean needSplit;
/** If we have a child SELECT that we should split. */
+ @GridToStringInclude
boolean needSplitChild;
/** If this is UNION ALL. Makes sense only for type UNION.*/
@@ -2003,6 +2020,11 @@ public class GridSqlQuerySplitter {
private boolean isQuery() {
return type == Type.SELECT || type == Type.UNION;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(QueryModel.class, this);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
index 21cf596..f3acfb9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java
@@ -31,6 +31,11 @@ public abstract class GridSqlStatement {
*/
public abstract String getSQL();
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return getSQL();
+ }
+
/**
* @param explain Explain.
* @return {@code this}.
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
index 887e427..f073e7d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java
@@ -41,7 +41,7 @@ public class GridSqlSubquery extends GridSqlElement {
/**
* @return Subquery AST.
*/
- public GridSqlQuery subquery() {
+ public <X extends GridSqlQuery> X subquery() {
return child(0);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
index f0ab123..07c8c5e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java
@@ -25,12 +25,10 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import junit.framework.AssertionFailedError;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheAtomicityMode;
@@ -39,7 +37,6 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.marshaller.optimized.OptimizedMarshaller;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -316,15 +313,7 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest
setDistributedJoins(distrib).
setEnforceJoinOrder(enforceJoinOrder)).getAll();
- try {
- assertRsEquals(h2Res, cacheRes, ordering);
- }
- catch (AssertionFailedError e) {
- X.println("Sql query:\n" + sql + "\nargs=" + Arrays.toString(args));
- X.println("[h2Res=" + h2Res + ", cacheRes=" + cacheRes + "]");
-
- throw e;
- }
+ assertRsEquals(h2Res, cacheRes, ordering);
return h2Res;
}
@@ -407,7 +396,9 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest
Iterator<Map.Entry<String,Integer>> iter1 = rowsWithCnt1.entrySet().iterator();
Iterator<Map.Entry<String,Integer>> iter2 = rowsWithCnt2.entrySet().iterator();
- for (;;) {
+ int uSize = rowsWithCnt1.size();
+
+ for (int i = 0;; i++) {
if (!iter1.hasNext()) {
assertFalse(iter2.hasNext());
@@ -419,8 +410,8 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest
Map.Entry<String, Integer> e1 = iter1.next();
Map.Entry<String, Integer> e2 = iter2.next();
- assertEquals(e1.getKey(), e2.getKey());
- assertEquals(e1.getValue(), e2.getValue());
+ assertEquals("Key " + i + " of " + uSize, e1.getKey(), e2.getKey());
+ assertEquals("Count " + i + " of " + uSize, e1.getValue(), e2.getValue());
}
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
index dc73e96..f8526a8 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java
@@ -31,8 +31,10 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.affinity.AffinityKey;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.typedef.X;
/**
* Executes one big query (and subqueries of the big query) to compare query results from h2 database instance and
@@ -54,7 +56,7 @@ import org.apache.ignite.configuration.CacheConfiguration;
* - date - | - date - |
* - alias - | - alias - |
* - archSeq - | - archSeq - ------------------- |
- * ------------------ | --------------------- ----repl.Exec------ |
+ * ------------------ | --------------------- ----part.Exec------ |
* | ------------------- |
* ----------------- | - rootOrderId PK - ----
* ---part.Cancel--- | - date -
@@ -119,19 +121,15 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
/** {@inheritDoc} */
@Override protected void setIndexedTypes(CacheConfiguration<?, ?> cc, CacheMode mode) {
- if (mode == CacheMode.PARTITIONED)
+ if (mode == CacheMode.PARTITIONED) {
cc.setIndexedTypes(
Integer.class, CustOrder.class,
useColocatedData() ? AffinityKey.class : Integer.class, ReplaceOrder.class,
useColocatedData() ? AffinityKey.class : Integer.class, OrderParams.class,
- useColocatedData() ? AffinityKey.class : Integer.class, Cancel.class
+ useColocatedData() ? AffinityKey.class : Integer.class, Cancel.class,
+ useColocatedData() ? AffinityKey.class : Integer.class, Exec.class
);
- else if (mode == CacheMode.REPLICATED)
- cc.setIndexedTypes(
- Integer.class, Exec.class
- );
- else
- throw new IllegalStateException("mode: " + mode);
+ }
}
/** {@inheritDoc} */
@@ -218,11 +216,12 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
int price = 1000 + rootOrderId;
int latsMkt = 3000 + rootOrderId;
- Exec exec = new Exec(rootOrderId, dates.get(rootOrderId % dates.size()), execShares, price, latsMkt);
+ Exec exec = new Exec(idGen.incrementAndGet(), rootOrderId,
+ dates.get(rootOrderId % dates.size()), execShares, price, latsMkt);
add(exec);
- rCache.put(exec.rootOrderId, exec);
+ pCache.put(exec.key(useColocatedData()), exec);
insertInDb(exec);
}
@@ -239,13 +238,20 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
"from \"part\".ReplaceOrder");
compareQueryRes0("select _key, _val, id, date, orderId, parentAlgo from \"part\".OrderParams\n");
compareQueryRes0("select _key, _val, id, date, refOrderId from \"part\".Cancel\n");
- compareQueryRes0(rCache, "select _key, _val, date, rootOrderId, execShares, price, lastMkt from \"repl\".Exec\n");
+ compareQueryRes0("select _key, _val, date, rootOrderId, execShares, price, lastMkt from \"part\".Exec\n");
}
/**
* @throws Exception If failed.
*/
public void testBigQuery() throws Exception {
+ X.println();
+ X.println(bigQry);
+ X.println();
+
+ X.println("Plan: " + pCache.query(new SqlFieldsQuery("EXPLAIN " + bigQry)
+ .setDistributedJoins(distributedJoins())).getAll());
+
List<List<?>> res = compareQueryRes0(pCache, bigQry, distributedJoins(), new Object[0], Ordering.RANDOM);
assertTrue(!res.isEmpty()); // Ensure we set good testing data at database.
@@ -301,9 +307,9 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
" refOrderId int" +
" )");
- st.execute("create table \"repl\".Exec" +
+ st.execute("create table \"part\".Exec" +
" (" +
- " _key int not null," +
+ " _key " + keyType + " not null," +
" _val other not null," +
" rootOrderId int unique," +
" date Date, " +
@@ -414,11 +420,11 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
*/
private void insertInDb(Exec o) throws SQLException {
try(PreparedStatement st = conn.prepareStatement(
- "insert into \"repl\".Exec (_key, _val, date, rootOrderId, execShares, price, lastMkt) " +
+ "insert into \"part\".Exec (_key, _val, date, rootOrderId, execShares, price, lastMkt) " +
"values(?, ?, ?, ?, ?, ?, ?)")) {
int i = 0;
- st.setObject(++i, o.rootOrderId);
+ st.setObject(++i, o.key(useColocatedData()));
st.setObject(++i, o);
st.setObject(++i, o.date);
st.setObject(++i, o.rootOrderId);
@@ -654,8 +660,12 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
* Execute information about root query.
*/
static class Exec implements Serializable {
- /** Primary key. */
+ /** */
@QuerySqlField
+ private int id;
+
+ /** */
+ @QuerySqlField(index = true)
private int rootOrderId;
/** Date */
@@ -675,13 +685,15 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
private int lastMkt;
/**
+ * @param id ID.
* @param rootOrderId Root order id.
* @param date Date.
* @param execShares Execute shares.
* @param price Price.
* @param lastMkt Last mkt.
*/
- Exec(int rootOrderId, Date date, int execShares, int price, int lastMkt) {
+ Exec(int id, int rootOrderId, Date date, int execShares, int price, int lastMkt) {
+ this.id = id;
this.rootOrderId = rootOrderId;
this.date = date;
this.execShares = execShares;
@@ -691,12 +703,16 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest {
/** {@inheritDoc} */
@Override public boolean equals(Object o) {
- return this == o || o instanceof Exec && rootOrderId == ((Exec)o).rootOrderId;
+ return this == o || o instanceof Exec && id == ((Exec)o).id;
}
/** {@inheritDoc} */
@Override public int hashCode() {
- return rootOrderId;
+ return id;
+ }
+
+ public Object key(boolean useColocatedData) {
+ return useColocatedData ? new AffinityKey<>(id, rootOrderId) : id;
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
index efa768c..8d42d44 100644
--- a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
+++ b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql
@@ -46,7 +46,7 @@ inner join (
select e.date, e.rootOrderId as eRootOrderId, e.rootOrderId, sum(e.execShares) as execShares,
sum(e.execShares*e.price)/sum(e.execShares) as price,
case when min(e.lastMkt) = max(e.lastMkt) then min(e.lastMkt) else min('XOFF') end as execMeet
- from "repl".Exec e
+ from "part".Exec e
group by e.date, e.rootOrderId
) oep on (cop.date = oep.date and cop.custOrderId = oep.eRootOrderId)
left outer join (